`scripts/scrape.mjs`

203 lines
#!/usr/bin/env node
/**
 * Gracie Studio scraper — FEED-FIRST, $0 plain-fetch.
 *
 * Gracie is a Next.js site over a Sanity CMS backend. There is NO usable
 * sitemap.xml (it returns the SPA HTML). The catalog is exposed as Next.js
 * page-data JSON:
 *   https://graciestudio.com/_next/data/<buildId>/collection/<slug>.json?slug=<slug>
 *
 * pageProps.collectionContent.wallpapers[] is the full product array.
 * Each wallpaper: title, subTitle (Gracie product code = mfr_sku), slug,
 * tags[], rightBody (rich-text description), image / hoverImage / imageList[]
 * (Sanity asset refs).
 *
 * Sanity image URL: https://cdn.sanity.io/images/<projectId>/<dataset>/<assetId>-<WxH>.<ext>
 * from ref  image-<assetId>-<WxH>-<ext>.
 *
 * The buildId is discovered live from the homepage __NEXT_DATA__, so a Gracie
 * redeploy (new buildId) does not break the scraper.
 *
 * Upserts into dw_unified.gracie_catalog keyed on mfr_sku (ON CONFLICT).
 * Cost: $0 (local) — pure HTTP fetch, no Browserbase / proxy / captcha / paid API.
 */
import { Client } from 'pg';

const UA =
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36';
const BASE = 'https://graciestudio.com';
const SANITY = { projectId: 'fiu2vnvt', dataset: 'production' };
const DELAY_MS = 1200; // polite delay between collection fetches

// Sections that are wallcovering scenic panels vs fabric vs prints.
// Gracie is quote-only bespoke; we treat all as 'wallcovering' EXCEPT the
// explicit fabric collection.
const FABRIC_COLLECTIONS = new Set(['gracie-fabric']);

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

async function fetchText(url) {
  const res = await fetch(url, { headers: { 'User-Agent': UA, Accept: 'application/json,text/html' } });
  if (!res.ok) throw new Error(`HTTP ${res.status} ${url}`);
  return res.text();
}

async function getBuildIdAndCollections() {
  const html = await fetchText(`${BASE}/`);
  const bid = (html.match(/"buildId":"([^"]+)"/) || [])[1];
  if (!bid) throw new Error('could not find Next.js buildId on homepage');
  const cols = [...new Set([...html.matchAll(/href="\/collection\/([^"]+)"/g)].map((m) => m[1]))];
  return { buildId: bid, collections: cols };
}

// image-<assetId>-<WxH>-<ext>  ->  cdn URL
function refToUrl(ref) {
  if (!ref || typeof ref !== 'string') return null;
  const m = ref.match(/^image-([a-f0-9]+)-(\d+x\d+)-(\w+)$/);
  if (!m) return null;
  const [, assetId, dims, ext] = m;
  return `https://cdn.sanity.io/images/${SANITY.projectId}/${SANITY.dataset}/${assetId}-${dims}.${ext}`;
}

function imgUrl(imgObj) {
  return refToUrl(imgObj?.asset?._ref);
}

// flatten Sanity portable-text rich body to a plain string
function flattenBody(body) {
  if (!Array.isArray(body)) return '';
  const parts = [];
  for (const blk of body) {
    for (const ch of blk.children || []) if (ch.text) parts.push(ch.text);
  }
  return parts.join(' ').replace(/\s+/g, ' ').trim();
}

// strip the boilerplate CTA sentences Gracie appends to every description
function cleanDescription(txt) {
  if (!txt) return '';
  return txt
    .replace(/Complete the form below[^.]*\./gi, '')
    .replace(/Click the link below[^.]*\./gi, '')
    .replace(/Typically stocked pre-painted[^.]*\./gi, '')
    .replace(/\s+/g, ' ')
    .trim();
}

// Banned-word rule: OUR titles/descriptions say "Wallcovering", never "Wallpaper".
function sanitize(txt) {
  if (!txt) return txt;
  return txt
    .replace(/\bwallpapers\b/gi, 'wallcoverings')
    .replace(/\bwallpaper\b/gi, 'wallcovering');
}

function parseWallpaper(w, collectionSlug, collectionTitle) {
  const mfr = (w.subTitle || w.slug?.current || w._id || '').trim();
  if (!mfr) return null;
  const pattern = (w.title || mfr).trim();
  const gallery = (w.imageList || []).map((im) => imgUrl(im)).filter(Boolean);
  const primary = imgUrl(w.image) || imgUrl(w.hoverImage) || gallery[0] || null;
  const rawDesc = flattenBody(w.rightBody);
  const desc = sanitize(cleanDescription(rawDesc));
  const tags = (w.tags || []).map((t) => String(t)).filter(Boolean);
  const productUrl = w.slug?.current
    ? `${BASE}/${w.slug.current}`
    : `${BASE}/collection/${collectionSlug}`;
  const isFabric = FABRIC_COLLECTIONS.has(collectionSlug);
  return {
    mfr_sku: mfr,
    pattern_name: sanitize(pattern),
    collection: sanitize(collectionTitle || collectionSlug),
    image_url: primary,
    product_url: productUrl,
    product_type: isFabric ? 'fabric' : 'wallcovering',
    material: 'Handpainted',
    description: desc,
    all_images: gallery.join('|'),
    gallery_images: gallery,
    specs: {
      gracie_id: w._id,
      subtitle: w.subTitle || null,
      tags,
      hover_image: imgUrl(w.hoverImage),
      created_at: w._createdAt,
      updated_at: w._updatedAt,
    },
    ai_tags: tags.length ? tags : null,
  };
}

async function main() {
  const { buildId, collections } = await getBuildIdAndCollections();
  console.log(`[gracie] buildId=${buildId}  collections=${collections.length}: ${collections.join(', ')}`);

  const pg = new Client({ connectionString: process.env.PG || 'postgresql://localhost/dw_unified' });
  await pg.connect();

  const seen = new Set();
  let upserts = 0;
  const perCol = {};

  for (const slug of collections) {
    const url = `${BASE}/_next/data/${buildId}/collection/${encodeURIComponent(slug)}.json?slug=${encodeURIComponent(slug)}`;
    let json;
    try {
      json = JSON.parse(await fetchText(url));
    } catch (e) {
      console.warn(`[gracie] SKIP ${slug}: ${e.message}`);
      continue;
    }
    const cc = json?.pageProps?.collectionContent;
    if (!cc) {
      console.warn(`[gracie] ${slug}: no collectionContent`);
      continue;
    }
    const wallpapers = cc.wallpapers || [];
    perCol[slug] = 0;
    for (const w of wallpapers) {
      const rec = parseWallpaper(w, slug, cc.title);
      if (!rec) continue;
      if (seen.has(rec.mfr_sku)) continue; // dedup within run (patterns repeat across collections)
      seen.add(rec.mfr_sku);
      await pg.query(
        `INSERT INTO gracie_catalog
          (mfr_sku, pattern_name, collection, image_url, product_url, product_type,
           material, description, all_images, gallery_images, specs, ai_tags, last_scraped, crawled_at)
         VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,NOW(),NOW())
         ON CONFLICT (mfr_sku) DO UPDATE SET
           pattern_name=EXCLUDED.pattern_name,
           collection=EXCLUDED.collection,
           image_url=COALESCE(EXCLUDED.image_url, gracie_catalog.image_url),
           product_url=EXCLUDED.product_url,
           product_type=EXCLUDED.product_type,
           material=EXCLUDED.material,
           description=EXCLUDED.description,
           all_images=EXCLUDED.all_images,
           gallery_images=EXCLUDED.gallery_images,
           specs=EXCLUDED.specs,
           ai_tags=COALESCE(EXCLUDED.ai_tags, gracie_catalog.ai_tags),
           last_scraped=NOW(), updated_at=NOW()`,
        [
          rec.mfr_sku, rec.pattern_name, rec.collection, rec.image_url, rec.product_url,
          rec.product_type, rec.material, rec.description, rec.all_images, rec.gallery_images,
          JSON.stringify(rec.specs), rec.ai_tags ? JSON.stringify(rec.ai_tags) : null,
        ]
      );
      upserts++;
      perCol[slug]++;
    }
    console.log(`[gracie] ${slug}: ${perCol[slug]} patterns (of ${wallpapers.length} raw)`);
    await sleep(DELAY_MS);
  }

  const { rows } = await pg.query('SELECT COUNT(*) c, COUNT(image_url) img FROM gracie_catalog');
  console.log(`[gracie] DONE. upserts this run=${upserts}. table total=${rows[0].c}, with image=${rows[0].img}. Cost: $0 (local)`);
  await pg.end();
}

main().catch((e) => {
  console.error('[gracie] FATAL', e);
  process.exit(1);
});