`recrawl-miltonking-stragglers.js`

119 lines
#!/usr/bin/env node
// ============================================================================
// Milton & King — targeted "stragglers" re-crawl driver
// ----------------------------------------------------------------------------
// Re-crawls ONLY the milton_king products whose colorway rows still lack a
// gallery (all_images / gallery_images empty) after the 2026-06-10 local
// backfill. Mirrors the recrawl-osborne.js pattern but uses Milton's
// WooCommerce Store API (HTTP-only, no Puppeteer) which already returns the
// full image array per product — the same feed the deployed scrape-miltonking.js
// uses. The deployed scraper ALREADY captures gallery_images correctly; these
// 272 product URLs simply predate that capture (last full scrape 2026-03-14)
// or are edge cases, so a surgical re-fetch fills them.
//
// SAFE: read-only HTTP GET against the vendor's public API + UPSERT into the
//       LOCAL milton_king_catalog. No Shopify, no prod, no destructive op.
//
// Usage (local Mac2):  node recrawl-miltonking-stragglers.js
//   reads recrawl-worklist.tsv (col 3 = product_url) → de-dupes URLs →
//   hits /wp-json/wc/store/v1/products?slug=<slug> → upserts colorway rows.
//
// After it finishes, re-run the all_images backfill SQL (see PARK file) to
// propagate the freshly-captured gallery_images into vendor_catalog.all_images.
// ============================================================================

const https = require('https');
const fs = require('fs');
const path = require('path');

const BASE_URL = 'https://www.miltonandking.com';
const API_PATH = '/wp-json/wc/store/v1/products';
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
const WORKLIST = path.join(__dirname, 'recrawl-worklist.tsv');

function httpGet(url) {
  return new Promise((resolve, reject) => {
    const u = new URL(url);
    const req = https.get(
      { hostname: u.hostname, path: u.pathname + u.search,
        headers: { 'User-Agent': UA, 'Accept': 'application/json' }, timeout: 30000 },
      (res) => {
        if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
          const loc = res.headers.location;
          return httpGet(loc.startsWith('http') ? loc : BASE_URL + loc).then(resolve).catch(reject);
        }
        let d = ''; res.on('data', c => d += c); res.on('end', () => resolve({ status: res.statusCode, body: d }));
      });
    req.on('error', reject);
    req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
  });
}
const sleep = (ms) => new Promise(r => setTimeout(r, ms + Math.floor(Math.random() * 200)));

// Extract the product slug from a /product/<slug>/ URL
function slugOf(productUrl) {
  const m = (productUrl || '').match(/\/product\/([^/?#]+)/);
  return m ? m[1] : null;
}

(async () => {
  if (!fs.existsSync(WORKLIST)) { console.error('missing', WORKLIST); process.exit(1); }
  const lines = fs.readFileSync(WORKLIST, 'utf8').trim().split('\n').slice(1); // skip header
  const slugs = [...new Set(lines.map(l => slugOf(l.split('\t')[2])).filter(Boolean))];
  console.log(`Worklist: ${lines.length} colorway rows → ${slugs.length} distinct product slugs to re-fetch`);

  // Lazy-require the local DB helpers ONLY when actually writing, so a dry
  // listing run needs no pg creds. On Kamatera use ./scraper-utils instead.
  let upsertProduct = null, closePool = null;
  const DRY = process.argv.includes('--dry');
  if (!DRY) {
    try {
      ({ upsertProduct, closePool } = require('./scraper-utils'));
    } catch (e) {
      console.error('No scraper-utils found next to this script. Run on Kamatera vendor-scrapers dir, or pass --dry to just verify feed reachability.');
      process.exit(2);
    }
  }

  let ok = 0, withGallery = 0, gone = 0, errs = 0, rowsUpserted = 0;
  for (let i = 0; i < slugs.length; i++) {
    const slug = slugs[i];
    const url = `${BASE_URL}${API_PATH}?slug=${encodeURIComponent(slug)}`;
    try {
      const resp = await httpGet(url);
      if (resp.status !== 200) { errs++; await sleep(600); continue; }
      const arr = JSON.parse(resp.body);
      if (!Array.isArray(arr) || arr.length === 0) { gone++; await sleep(300); continue; }
      const p = arr[0];
      const images = (p.images || []).map(im => im.src).filter(Boolean);
      ok++;
      if (images.length >= 1) withGallery++;
      if (!DRY && upsertProduct) {
        // Upsert one row per colorway (same shape parseProduct builds in scrape-miltonking.js)
        const baseSku = (p.sku || '').trim();
        const colorAttr = (p.attributes || []).find(a => a.name === 'Color' || a.taxonomy === 'pa_color');
        const terms = colorAttr ? (colorAttr.terms || []) : [];
        const mkRow = (mfrSku, colorName, colorImages) => ({
          mfr_sku: mfrSku, pattern_name: (p.name || '').replace(/\s+Wallpaper$/i, '').trim(),
          color_name: colorName || null, product_url: p.permalink || `${BASE_URL}/product/${slug}/`,
          image_url: (colorImages[0] || images[0]) || null,
          gallery_images: (colorImages.length ? colorImages : images).length
            ? `{${(colorImages.length ? colorImages : images).map(u => `"${u}"`).join(',')}}` : null,
        });
        const targets = terms.length
          ? terms.map(t => {
              const cn = (t.name || '').trim(), cs = (t.slug || '').toLowerCase();
              const ci = (p.images || []).filter(im => ((im.alt || '').toLowerCase().includes(cn.toLowerCase()) || (im.name || '').toLowerCase().includes(cs))).map(im => im.src);
              return mkRow(baseSku ? `${baseSku}-${cs}` : `mk-${p.id}-${cs}`, cn, ci);
            })
          : [mkRow(baseSku || `mk-${p.id}`, null, [])];
        for (const r of targets) { const id = await upsertProduct('milton_king_catalog', r); if (id) rowsUpserted++; }
      }
      if ((i + 1) % 25 === 0) console.log(`  ${i + 1}/${slugs.length} | ok=${ok} gallery=${withGallery} gone=${gone} errs=${errs} upserted=${rowsUpserted}`);
      await sleep(300);
    } catch (e) { errs++; await sleep(800); }
  }
  console.log(`\nDONE. slugs=${slugs.length} ok=${ok} withGallery=${withGallery} gone(discontinued)=${gone} errs=${errs} rowsUpserted=${rowsUpserted}`);
  if (closePool) await closePool();
})().catch(e => { console.error('FATAL', e); process.exit(1); });