`scrape.mjs`

163 lines
#!/usr/bin/env node
// Crezana Design feed-first scraper — GoDaddy Website Builder (DPS) editorial galleries.
// $0 plain-fetch. No SKUs/prices on the site → derive deterministic mfr_sku from image hash.
// Vendor-scoped to Crezana ONLY. product_type = 'Fabric'. QUOTE-ONLY.
import pg from 'pg';

const SITE = 'https://crezanadesign.com';
const SITE_UUID = '1f48ce84-09d7-40fc-8a82-23198f7239f7';
const PGHOST = process.env.PGHOST || '/tmp';

// Category pages discovered from sitemap.website.xml. Path -> friendly collection label.
const CATEGORIES = [
  { path: '/',              collection: 'Home' },
  { path: '/printed',       collection: 'Printed' },
  { path: '/textures',      collection: 'Textures' },
  { path: '/embroideries',  collection: 'Embroideries' },
  { path: '/embroideries-ii',collection: 'Embroideries II' },
  { path: '/madagascars',   collection: 'Madagascars' },
  { path: '/grommets%2Fstuds', collection: 'Grommets & Studs' },
  { path: '/showrooms',     collection: 'Showrooms' },
];

async function fetchText(url) {
  const r = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36' } });
  if (!r.ok) throw new Error(`${r.status} ${url}`);
  return await r.text();
}

// GoDaddy stores captions as escaped DraftJS JSON. Decode to plain text.
function decodeCaption(escaped) {
  if (!escaped) return '';
  try {
    // The JS bundle double-escapes; unescape backslashes then parse.
    let s = escaped.replace(/\\"/g, '"').replace(/\\\\/g, '\\');
    const j = JSON.parse(s);
    if (j && j.blocks) return j.blocks.map(b => b.text).join(' ').trim();
  } catch { /* fall through */ }
  return '';
}

// Boilerplate disclaimer text we want to ignore as a "name".
const BOILERPLATE = /inconsistencies may occur|dye lot|request sample|patterns and colors are approximate/i;

// Parse a gpub script bundle for gallery images with captions.
// Structure (escaped): "galleryImages":[{"image":{... "image":"//img1.wsimg.com/isteam/ip/UUID/HASH.jpg" ...},"caption":"{draftjs}"}]
function parseGpub(js) {
  const out = [];
  // Find each image object + its following caption.
  // Match: "image":"//img1.wsimg.com/isteam/ip/UUID/HASH.ext"  ... "caption":"..."
  const imgRe = new RegExp('\\\\?"image\\\\?":\\\\?"(\\/\\/img1\\.wsimg\\.com\\/isteam\\/ip\\/' + SITE_UUID + '\\/[a-f0-9]+\\.(?:jpg|jpeg|png|webp))\\\\?"', 'gi');
  let m;
  const seen = new Set();
  while ((m = imgRe.exec(js)) !== null) {
    let url = m[1];
    if (seen.has(url)) continue;
    seen.add(url);
    // Try to grab a caption within the next ~600 chars after this image.
    const tail = js.slice(m.index, m.index + 1200);
    const capM = tail.match(/\\?"caption\\?":\\?"(\{.*?entityMap.*?\})\\?"/);
    let caption = '';
    if (capM) caption = decodeCaption(capM[1]);
    if (BOILERPLATE.test(caption)) caption = '';
    out.push({ url: 'https:' + url, caption });
  }
  return out;
}

function hashFromUrl(url) {
  const m = url.match(/\/([a-f0-9]{16,})\.(?:jpg|jpeg|png|webp)/i);
  return m ? m[1] : null;
}

async function main() {
  const items = []; // {mfr_sku, collection, image_url, product_url, caption}
  const globalSeenHash = new Set();

  for (const cat of CATEGORIES) {
    const pageUrl = SITE + cat.path;
    let html;
    try { html = await fetchText(pageUrl); }
    catch (e) { console.error(`  ! page ${cat.path}: ${e.message}`); continue; }

    const hashes = [...html.matchAll(/gpub\/([a-f0-9]+)\/script\.js/g)].map(x => x[1]);
    const uniqHashes = [...new Set(hashes)];
    let catImages = [];
    for (const h of uniqHashes) {
      try {
        const js = await fetchText(`https://img1.wsimg.com/blobby/go/${SITE_UUID}/gpub/${h}/script.js`);
        catImages = catImages.concat(parseGpub(js));
      } catch (e) { console.error(`    ! gpub ${h}: ${e.message}`); }
    }

    let n = 0;
    for (const img of catImages) {
      const hash = hashFromUrl(img.url);
      if (!hash) continue;
      if (globalSeenHash.has(hash)) continue; // an image appears in one canonical collection only
      globalSeenHash.add(hash);
      n++;
      const mfr_sku = `CREZ-${hash.slice(0, 12).toUpperCase()}`;
      const collSlug = cat.collection.replace(/[^A-Za-z0-9]+/g, '-').replace(/-+$/,'');
      const pattern_name = img.caption && img.caption.length > 2 && img.caption.length < 120
        ? img.caption
        : `Crezana ${cat.collection} ${String(n).padStart(3, '0')}`;
      items.push({
        mfr_sku,
        pattern_name,
        collection: cat.collection,
        image_url: img.url,
        product_url: pageUrl,
      });
    }
    console.log(`  ${cat.collection}: ${n} images`);
  }

  console.log(`\nTotal unique gallery images: ${items.length}`);

  // Upsert
  const client = new pg.Client({ host: PGHOST, database: 'dw_unified' });
  await client.connect();
  let ins = 0, upd = 0;
  for (const it of items) {
    const r = await client.query(
      `INSERT INTO crezana_catalog (mfr_sku, pattern_name, collection, image_url, product_url, product_type, crawled_at, last_scraped, updated_at)
       VALUES ($1,$2,$3,$4,$5,'Fabric',now(),now(),now())
       ON CONFLICT (mfr_sku) DO UPDATE SET
         pattern_name=EXCLUDED.pattern_name,
         collection=EXCLUDED.collection,
         image_url=EXCLUDED.image_url,
         product_url=EXCLUDED.product_url,
         last_scraped=now(),
         updated_at=now()
       RETURNING (xmax=0) AS inserted`,
      [it.mfr_sku, it.pattern_name, it.collection, it.image_url, it.product_url]
    );
    if (r.rows[0].inserted) ins++; else upd++;
  }

  // Assign DW series SKUs (DWCZ- + 6-digit number) to any rows still missing one —
  // continue from the current max so the series never regresses to NULL and never
  // collides. HARD RULE (Steve): DWCZ-###### 6-digit numbers ONLY, never collection
  // letters. Idempotent: a normal re-run touches 0 rows.
  const dw = await client.query(`
    WITH mx AS (
      SELECT COALESCE(MAX((regexp_replace(dw_sku,'^DWCZ-',''))::int), 890999) AS n
      FROM crezana_catalog WHERE dw_sku ~ '^DWCZ-[0-9]{6}$'
    ),
    todo AS (
      SELECT id, (SELECT n FROM mx) + row_number() OVER (ORDER BY id) AS seq
      FROM crezana_catalog WHERE dw_sku IS NULL
    )
    UPDATE crezana_catalog c
    SET dw_sku = 'DWCZ-' || todo.seq, updated_at = now()
    FROM todo WHERE c.id = todo.id
    RETURNING c.id`);
  if (dw.rowCount) console.log(`Assigned ${dw.rowCount} new DWCZ-###### series SKU(s).`);

  await client.end();
  console.log(`Upserted: ${ins} inserted, ${upd} updated. Cost: $0 (local)`);
}

main().catch(e => { console.error(e); process.exit(1); });