`recrawl-cowtan.js`

163 lines
#!/usr/bin/env node
// recrawl-cowtan.js — additive all_images + full-spec backfill for cowtan_tout.
//
// WHY THIS EXISTS: designs.cowtan.com re-platformed to an Alpine.js SPA after the
// Apr-30 crawl. The old `POST /Search` AJAX endpoint that scrape-cowtan.js depends on
// now returns HTTP 404, so the deployed scraper is broken AND it never captured
// all_images in the first place (it built a single image_url from StockCode).
//
// This driver uses the NEW, verified feed-first API:
//   GET /api/Product/Detail/<StockCode>  ->  JSON array of sibling colourways, each with
//     full specs (Width, RepeatV/H, Composition, Qualities, Price, InStock, Type) and
//     RoomShotPictureIDs (room-scene image IDs).
// Hi-res flatshot per colourway is built from the SPA's own template:
//   https://d2mq91o692rj7w.cloudfront.net/flatshots/2017-main/<urlSafeStockCode>_m.jpg
// (thumb = 2017-thumb/<code>_t.jpg). urlSafeStockCode = StockCode.replaceAll('/','-').
//
// all_images per row = JSON array of [ main flatshot, ...room-shot urls ] for THAT colourway.
// Each sibling colourway is its own mfr_sku row and gets its own all_images.
//
// ADDITIVE & IDEMPOTENT: only UPDATEs all_images / specs columns on existing rows
// (matched by mfr_sku). Never deletes, never inserts net-new SKUs, never archives.
// One Detail fetch covers a whole pattern (all its colourways), so we fetch by distinct
// ProductCode to minimise requests (~one call per pattern, not per colourway).
//
// USAGE (run ON Kamatera prod, gated — see pending-approval/cowtan_tout-recrawl.md):
//   node recrawl-cowtan.js --net-new --all-images           # backfill rows missing all_images
//   node recrawl-cowtan.js --net-new --all-images --limit 50 --dry-run   # safe probe
//
// ROOM-SHOT URL PATTERN: RoomShotPictureIDs was empty on the probed samples. Before the
// full prod run, confirm the room-shot CDN path on a SKU that HAS room shots and set
// ROOMSHOT_URL() below. Until confirmed, room shots are skipped (main flatshot still
// populates all_images, clearing the gate). This is the one item to verify when attended.

const https = require('https');

// ---- config -------------------------------------------------------------
const TABLE = process.env.COWTAN_TABLE || 'vendor_catalog';       // local: vendor_catalog (vendor_code filter); prod fleet table may be cowtan_tout_catalog
const VENDOR_CODE = 'cowtan_tout';
const CDN = 'https://d2mq91o692rj7w.cloudfront.net/flatshots/';
const API = 'https://designs.cowtan.com';
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
const CONCURRENCY = 3;        // polite — vendor is the same one we re-platformed off of
const SLEEP_MS = 350;

const argv = process.argv.slice(2);
const DRY = argv.includes('--dry-run');
const LIMIT = (() => { const i = argv.indexOf('--limit'); return i > -1 ? parseInt(argv[i + 1], 10) : 0; })();

// pg pool: prefer the fleet scraper-utils pool if present (prod), else local DATABASE_URL.
let pool;
try {
  ({ pool } = require('/root/DW-Agents/vendor-scrapers/scraper-utils'));
} catch (_) {
  const { Pool } = require('pg');
  pool = new Pool({ connectionString: process.env.DATABASE_URL || 'postgres://stevestudio2@localhost/dw_unified' });
}

// ---- helpers ------------------------------------------------------------
const sleep = (ms) => new Promise(r => setTimeout(r, ms + Math.floor(Math.random() * 200)));
const urlSafe = (code) => String(code).replaceAll('/', '-');
const mainFlatshot = (code) => CDN + '2017-main/' + urlSafe(code) + '_m.jpg';
// CONFIRM-WHEN-ATTENDED: room-shot URL builder. Returns [] until the pattern is verified.
const ROOMSHOT_URL = (picId) => null; // e.g. CDN + 'rooms/' + picId + '.jpg' — TBD, see header note.

function getJson(path, maxRedirects = 4) {
  return new Promise((resolve, reject) => {
    const req = https.request({ hostname: 'designs.cowtan.com', path, method: 'GET', timeout: 25000,
      headers: { 'User-Agent': UA, 'Accept': 'application/json, */*', 'X-Requested-With': 'XMLHttpRequest' } }, res => {
      if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location && maxRedirects > 0) {
        res.resume();
        const loc = res.headers.location.replace(API, '');
        return getJson(loc, maxRedirects - 1).then(resolve).catch(reject);
      }
      if (res.statusCode !== 200) { res.resume(); return reject(new Error('HTTP ' + res.statusCode)); }
      let d = ''; res.on('data', c => d += c); res.on('end', () => { try { resolve(JSON.parse(d)); } catch (e) { reject(new Error('JSON ' + e.message)); } });
    });
    req.on('error', reject); req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
    req.end();
  });
}

function buildAllImages(entry) {
  const imgs = [mainFlatshot(entry.StockCode)];
  for (const pid of (entry.RoomShotPictureIDs || [])) { const u = ROOMSHOT_URL(pid); if (u) imgs.push(u); }
  return [...new Set(imgs)];
}

function specJson(entry) {
  return {
    width: entry.Width || null, repeat_v: entry.RepeatV || null, repeat_h: entry.RepeatH || null,
    finish: entry.Finish || null, type: entry.Type || null, weight: entry.Weight || null,
    composition: entry.Composition || null, qualities: entry.Qualities || null,
    testing: entry.Testing || null, price: entry.Price || null, in_stock: entry.InStock ?? null,
    availability: entry.Availability || null, pattern_book: entry.PatternBook || null,
    brand_name: entry.BrandName || null, room_shot_picture_ids: entry.RoomShotPictureIDs || [],
  };
}

// ---- main ---------------------------------------------------------------
(async () => {
  // worklist: rows still missing all_images. We crawl by distinct ProductCode (pattern)
  // since one Detail call returns every colourway of that pattern.
  // Self-heal clause: also re-select rows whose all_images was previously written with a
  // 3rd-party share-link leak (pinterest/facebook), so a bad earlier run cures itself.
  // Images are built host-anchored to cowtan's own CDN, so a clean re-run overwrites junk.
  const contam = `all_images ILIKE '%pinterest%' OR all_images ILIKE '%facebook%'`;
  const where = `vendor_code = $1 AND (all_images IS NULL OR all_images = '' OR all_images = '[]' OR all_images = 'null' OR ${contam})`;
  const q = `SELECT DISTINCT split_part(mfr_sku,'-',1) AS product_code, min(mfr_sku) AS seed_sku
             FROM ${TABLE} WHERE ${where} GROUP BY 1 ORDER BY 1` + (LIMIT ? ` LIMIT ${LIMIT}` : '');
  const { rows } = await pool.query(q, [VENDOR_CODE]).catch(async (e) => {
    // fleet per-vendor table has no vendor_code column — fall back to plain table scan.
    if (/vendor_code/.test(e.message)) {
      const w2 = `(all_images IS NULL OR all_images='' OR all_images='[]' OR all_images='null' OR all_images ILIKE '%pinterest%' OR all_images ILIKE '%facebook%')`;
      return pool.query(`SELECT DISTINCT split_part(mfr_sku,'-',1) AS product_code, min(mfr_sku) AS seed_sku FROM ${TABLE} WHERE ${w2} GROUP BY 1 ORDER BY 1` + (LIMIT ? ` LIMIT ${LIMIT}` : ''));
    }
    throw e;
  });

  console.log(`Re-crawling ${rows.length} distinct cowtan patterns (table=${TABLE}, dry=${DRY})...`);
  let patterns = 0, skusUpdated = 0, errs = 0, idx = 0;

  async function worker() {
    while (idx < rows.length) {
      const i = idx++;
      const seed = rows[i].seed_sku;
      try {
        const detail = await getJson('/api/Product/Detail/' + encodeURIComponent(seed));
        const entries = Array.isArray(detail) ? detail : Object.values(detail);
        for (const e of entries) {
          if (!e || !e.StockCode) continue;
          const all = JSON.stringify(buildAllImages(e));
          const specs = JSON.stringify(specJson(e));
          if (DRY) { skusUpdated++; if (skusUpdated <= 5) console.log('  [dry]', e.StockCode, all); continue; }
          // additive UPDATE only — never insert/delete. Match by mfr_sku within this vendor.
          const r = await pool.query(
            `UPDATE ${TABLE} SET all_images = $1, specs = COALESCE(specs,'{}'::jsonb) || $2::jsonb, updated_at = NOW()
             WHERE mfr_sku = $3` + (/vendor_catalog/.test(TABLE) ? ` AND vendor_code = '${VENDOR_CODE}'` : ''),
            [all, specs, e.StockCode]
          ).catch(async (err) => {
            // table may not have a jsonb specs column — degrade to all_images-only.
            if (/column .*specs/.test(err.message)) {
              return pool.query(`UPDATE ${TABLE} SET all_images = $1, updated_at = NOW() WHERE mfr_sku = $2` + (/vendor_catalog/.test(TABLE) ? ` AND vendor_code='${VENDOR_CODE}'` : ''), [all, e.StockCode]);
            }
            throw err;
          });
          if (r.rowCount) skusUpdated += r.rowCount;
        }
        patterns++;
        if (patterns % 50 === 0) console.log(`  ${patterns}/${rows.length} patterns | ${skusUpdated} skus updated | ${errs} errs`);
        await sleep(SLEEP_MS);
      } catch (e) {
        errs++;
        if (errs % 10 === 0) console.error(`  err at ${seed}: ${e.message}`);
        await sleep(800);
      }
    }
  }

  await Promise.all(Array.from({ length: CONCURRENCY }, worker));
  console.log(`\nDONE. patterns=${patterns} sku_rows_updated=${skusUpdated} errors=${errs}`);
  await (pool.end ? pool.end() : Promise.resolve());
})().catch(e => { console.error('FATAL', e); (pool.end ? pool.end() : Promise.resolve()).finally(() => process.exit(1)); });