`recrawl-designers-guild.js`

175 lines
#!/usr/bin/env node
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
// ============================================================================
// Designers Guild ADDITIVE all_images re-crawl — Codename: Diana (full-page pass)
// ============================================================================
// WHY: the deployed scrape-designers-guild.js captures only ONE image per
// product — the first `imageId` (→ /image-nonwebp/600/<id>), with og:image as a
// fallback. Steve's standing rule "always pull all data and images" requires
// every product to carry its FULL image set (cutting + roomsets + per-colourway
// swatches) before it is import-ready. As of 2026-06-10 ALL 2,163 designers_guild
// rows in vendor_catalog have an empty all_images. This driver re-crawls the
// actual product PAGE for every eligible row and fills all_images additively.
//
// IMAGE SCOPING (confirmed against live HTML 2026-06-10):
//   A Designers Guild product page is a ~900KB SSR JSON page that also embeds
//   thumbnails of *related* products. The product's OWN gallery is exactly the
//   set of imageIds that the page references at the full /image-nonwebp/1024/<id>
//   size — related-product cards only ever appear at the 156/239 thumb sizes.
//   So: collect every distinct id in `/image-nonwebp/1024/<id>`; if none are
//   present, fall back to the ids inside the product's "images":[{imageId}] JSON
//   arrays; final fallback = the existing hero image_url. Each id is stored as a
//   canonical https://www.designersguild.com/image-nonwebp/1024/<id> URL.
//
// SAFETY / ADDITIVE-ONLY:
//   * NEVER inserts rows. UPDATE only, matched by (vendor_code,mfr_sku).
//   * Only touches rows whose all_images is empty AND whose product_url is a real
//     designersguild.com page. The 869 URL-less rows (all in the already-on-Shopify
//     / dup cohort) are skipped — they are dedup'd out at publish anyway.
//   * Does NOT push to Shopify, does NOT publish, does NOT change SKUs.
//   * Targets vendor_catalog (vendor_code='designers_guild') — the unified table
//     where the all_images-completeness + never-duplicate gates actually read.
//
// MODES:
//   --canary N    re-crawl only the first N rows (default 5) — RUN THIS FIRST to
//                 confirm the /1024/ gallery heuristic against live HTML.
//   --all         re-crawl every eligible row (the full pass).
//   --net-new     restrict to rows whose mfr_sku is NOT already in shopify_products
//                 (the genuine net-net-new push-prep cohort: 985 rows on 2026-06-10).
//
// Mirrors the harness of the FIXED recrawl-osborne.js / recrawl-coordonne.js
// (httpGet + worker pool). Designed to run ON Kamatera (/root paths, scraper-utils).
// ============================================================================
require('dotenv').config({ path: '/root/DW-Agents/dw-price-stock/.env' });
const https = require('https');
const { pool } = require('/root/DW-Agents/vendor-scrapers/scraper-utils');

const VENDOR_CODE = 'designers_guild';
const SITE_BASE = 'https://www.designersguild.com';
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';

const argv = process.argv.slice(2);
const FULL = argv.includes('--all');
const NET_NEW_ONLY = argv.includes('--net-new');
const canaryIdx = argv.indexOf('--canary');
const CANARY_N = canaryIdx >= 0 ? (parseInt(argv[canaryIdx + 1], 10) || 5) : (FULL ? 0 : 5);

function httpGet(url, maxRedirects = 5) {
  return new Promise((resolve, reject) => {
    const req = https.get(url, {
      timeout: 30000,
      headers: {
        'User-Agent': UA,
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.9',
        'Cookie': 'country=US; currency=USD',
      },
    }, (res) => {
      if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location && maxRedirects > 0) {
        const loc = res.headers.location.startsWith('http') ? res.headers.location : SITE_BASE + res.headers.location;
        res.resume();
        return httpGet(loc, maxRedirects - 1).then(resolve).catch(reject);
      }
      if (res.statusCode !== 200) { res.resume(); return reject(new Error('HTTP ' + res.statusCode)); }
      let d = ''; res.on('data', c => d += c); res.on('end', () => resolve(d)); res.on('error', reject);
    });
    req.on('error', reject);
    req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
  });
}
const sleep = (ms) => new Promise(r => setTimeout(r, ms + Math.floor(Math.random() * 250)));

const decode = (s) => s
  .replace(/&quot;/g, '"').replace(/&#34;/g, '"')
  .replace(/&amp;/g, '&').replace(/&#38;/g, '&');

// Extract the product's OWN gallery imageIds, scoped via the /1024/ full-size
// heuristic, with the embedded "images":[{imageId}] JSON arrays as a fallback.
function extractImageIds(html) {
  const ids = [];
  const seen = new Set();
  const add = (id) => { if (id && !seen.has(id)) { seen.add(id); ids.push(id); } };

  // 1) Canonical: every full-size /image-nonwebp/1024/<id> the product references.
  let m;
  const reFull = /\/image-nonwebp\/1024\/(\d+)/g;
  while ((m = reFull.exec(html))) add(m[1]);

  // 2) Fallback: ids inside the product's "images":[{"imageId":N}] JSON arrays
  //    (HTML-entity-encoded in the SSR payload).
  if (!ids.length) {
    const d = decode(html);
    const reArr = /"images":\[(.*?)\]/g;
    let a;
    while ((a = reArr.exec(d))) {
      let im; const reId = /"imageId":(\d+)/g;
      while ((im = reId.exec(a[1]))) add(im[1]);
    }
  }
  return ids;
}

const idToUrl = (id) => `${SITE_BASE}/image-nonwebp/1024/${id}`;

(async () => {
  let sql = `
    SELECT mfr_sku, split_part(product_url,'?',1) AS url, image_url
    FROM vendor_catalog
    WHERE vendor_code = '${VENDOR_CODE}'
      AND (all_images IS NULL OR all_images = '' OR all_images = '[]')
      AND product_url ~ 'designersguild\\.com'
  `;
  if (NET_NEW_ONLY) {
    sql += ` AND upper(btrim(mfr_sku)) NOT IN
             (SELECT upper(btrim(mfr_sku)) FROM shopify_products WHERE mfr_sku IS NOT NULL AND mfr_sku <> '')`;
  }
  sql += ` ORDER BY mfr_sku`;
  if (CANARY_N > 0) sql += ` LIMIT ${CANARY_N}`;

  const { rows } = await pool.query(sql);
  console.log(`Designers Guild additive all_images re-crawl — ${rows.length} pages${CANARY_N > 0 ? ` (CANARY/limit ${CANARY_N})` : ' (FULL)'}${NET_NEW_ONLY ? ' [net-new only]' : ''}`);

  let pages = 0, updated = 0, withImgs = 0, multiImg = 0, noImgs = 0, errs = 0;
  const concurrency = 3;
  let idx = 0;
  async function worker() {
    while (idx < rows.length) {
      const i = idx++;
      const { mfr_sku, url, image_url } = rows[i];
      try {
        const html = await httpGet(url);
        let ids = extractImageIds(html);
        let images = ids.map(idToUrl);
        // final fallback: keep the existing hero if the page yielded nothing
        if (!images.length && image_url) images = [image_url];
        pages++;
        if (!images.length) { noImgs++; await sleep(300); continue; }
        await pool.query(
          `UPDATE vendor_catalog
             SET all_images = $1,
                 image_url  = COALESCE(NULLIF(image_url,''), $2),
                 last_scraped_at = NOW()
           WHERE vendor_code = $3 AND mfr_sku = $4`,
          [JSON.stringify(images), images[0], VENDOR_CODE, mfr_sku]
        );
        updated++; withImgs++;
        if (images.length > 1) multiImg++;
        if (CANARY_N > 0) {
          console.log(`  ${mfr_sku}: ${images.length} imgs`);
          console.log(`    ${images.slice(0, 4).join('\n    ')}${images.length > 4 ? '\n    …' : ''}`);
        } else if (pages % 50 === 0) {
          console.log(`  ${pages}/${rows.length} | updated ${updated} | w/imgs ${withImgs} | multi ${multiImg} | noImgs ${noImgs} | errs ${errs}`);
        }
        await sleep(350);
      } catch (e) {
        errs++;
        if (errs % 10 === 0 || CANARY_N > 0) console.error(`  err ${mfr_sku} ${url}: ${e.message}`);
        await sleep(800);
      }
    }
  }
  await Promise.all(Array.from({ length: concurrency }, worker));
  console.log(`\nDONE. pages=${pages} updated=${updated} with_imgs=${withImgs} multi_image=${multiImg} no_imgs=${noImgs} errors=${errs}`);
  await pool.end();
})().catch(e => { console.error('FATAL', e); pool.end().catch(() => {}); process.exit(1); });