`scraper/scrape-atomic50.mjs`

160 lines
#!/usr/bin/env node
/**
 * scrape-atomic50.mjs — feed-first Atomic 50 Ceilings refresh.
 *
 * Atomic 50 is a Squarespace site. Despite the skill note ("requires headless
 * browser"), it exposes the feed-first crack: sitemap.xml lists every INDIVIDUAL
 * product URL, and each returns full structured JSON at `<url>?format=json`.
 * $0 plain-fetch — NO Puppeteer, NO captcha, NO login.
 *
 * CATALOG SHAPE (important): atomic50_catalog AT50-* rows are TWO kinds:
 *   1. individual product pages (/at50-24, /at50-c1, /at50-m11, /at50g34) — crawlable.
 *   2. accessory/molding line-items (SP1..SP9, NAIL, SNIP, CLIP, TCRO, APAD, ...) that
 *      ALL share ONE page (/molding-accessories) — NOT individually crawlable.
 * So matching + disco are scoped to INDIVIDUAL pages, matched BY product_url (slug->mfr_sku
 * is unreliable: /at50g34 vs catalog AT50-G34). Accessory-page rows are never disco-touched.
 *
 * Behaviour (idempotent, NON-DESTRUCTIVE):
 *   - UPSERT by product_url: existing individual page -> refresh all_images/image_url/
 *     last_scraped, discontinued=false; PRESERVE shopify_product_id, on_shopify,
 *     product_type, dw_sku, ai_*. New individual page -> INSERT with next DWJT-6000xx.
 *   - DISCO: an individual-page catalog row NOT seen in the sitemap is re-checked with a
 *     live GET; only a real 404/410 sets discontinued=true. Never deletes. Guarded by a
 *     70%-of-individual-count floor (transient-hiccup guard). Set env DISCO=0 to skip.
 *
 * Cost: $0 (local PG socket + plain HTTPS). Never writes Shopify.
 */
import { createRequire } from 'node:module';
const require = createRequire(import.meta.url);
const { Client } = require('pg');

const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36';
const SITEMAP = 'https://www.atomic50ceilings.com/sitemap.xml';
const IMG_CAP = 40;
const DISCO_ON = process.env.DISCO !== '0';
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const normUrl = (u) => String(u || '').trim().replace(/\/+$/, '').replace(/\?.*$/, '').toLowerCase();

async function fetchText(url, tries = 4) {
  for (let a = 0; a < tries; a++) {
    try { const r = await fetch(url, { headers: { 'User-Agent': UA } }); if (r.ok) return await r.text(); }
    catch (_) { /* retry */ }
    await sleep(800 * (a + 1));
  }
  throw new Error('fetch failed: ' + url);
}
async function statusOf(url) {
  try { const r = await fetch(url, { headers: { 'User-Agent': UA } }); return r.status; }
  catch (_) { return 0; }
}

function extractImages(rawJson) {
  const out = [], seen = new Set();
  const re = /https:(?:\\\/|\/){2}images\.squarespace-cdn\.com(?:\\\/|\/)[^"'\\)\s]+/g;
  let m;
  while ((m = re.exec(rawJson))) {
    let u = m[0].replace(/\\\//g, '/').replace(/\?.*$/, '') + '?format=2500w';
    const key = u.replace(/\?format=2500w$/, '');
    if (u.includes('/content/v1/') && !seen.has(key)) { seen.add(key); out.push(u); }
    if (out.length >= IMG_CAP) break;
  }
  return out;
}
function extractTitle(rawJson, slug) {
  const m = rawJson.match(/"(?:title|websiteTitle|seoTitle)"\s*:\s*"([^"]{2,120})"/);
  if (m && m[1] && !/atomic\s*50/i.test(m[1])) return m[1];
  return slug.toUpperCase();
}

async function main() {
  const t0 = Date.now();
  const xml = await fetchText(SITEMAP);
  const locs = [...xml.matchAll(/<loc>([^<]+)<\/loc>/g)].map((m) => m[1]);
  const products = [];
  const seenUrl = new Set();
  for (const u of locs) {
    if (!/\/at50[-a-z0-9]+$/i.test(u)) continue;          // individual product pages (incl. at50g34)
    const nu = normUrl(u);
    if (seenUrl.has(nu)) continue; seenUrl.add(nu);
    products.push({ url: u, nurl: nu, slug: u.split('/').pop() });
  }
  console.log(`sitemap: ${locs.length} locs -> ${products.length} individual at50 product pages`);

  const scraped = [];
  for (const p of products) {
    try {
      const raw = await fetchText(p.url + '?format=json');
      scraped.push({ ...p, images: extractImages(raw), title: extractTitle(raw, p.slug) });
    } catch (e) { console.log(`  ! skip ${p.slug}: ${e.message}`); }
    await sleep(150);
  }
  const scrapedByUrl = new Map(scraped.map((s) => [s.nurl, s]));

  const db = new Client({ host: '/tmp', database: 'dw_unified' });
  await db.connect();

  const existing = (await db.query(
    `SELECT dw_sku, mfr_sku, product_url FROM atomic50_catalog WHERE mfr_sku IS NOT NULL`
  )).rows;
  // individual-page catalog rows = those whose product_url is an /at50* page (not /molding-accessories)
  const indivRows = existing.filter((r) => /\/at50[-a-z0-9]+$/i.test(r.product_url || ''));
  const haveByUrl = new Map(indivRows.map((r) => [normUrl(r.product_url), r]));

  const maxRow = await db.query(
    `SELECT MAX((regexp_replace(dw_sku,'\\D','','g'))::bigint) AS n FROM atomic50_catalog WHERE dw_sku ~ '^DWJT-'`
  );
  let nextNum = (Number(maxRow.rows[0].n) || 600059) + 1;

  let inserted = 0, updated = 0;
  for (const s of scraped) {
    const imgsJson = JSON.stringify(s.images);
    const firstImg = s.images[0] || null;
    const row = haveByUrl.get(s.nurl);
    if (row) {
      await db.query(
        `UPDATE atomic50_catalog SET all_images=$2::jsonb, image_url=COALESCE($3,image_url),
             discontinued=false, last_scraped=now(), updated_at=now() WHERE dw_sku=$1`,
        [row.dw_sku, imgsJson, firstImg]
      );
      updated++;
    } else {
      const dw = `DWJT-${nextNum++}`;
      const mfr = s.slug.toUpperCase();
      await db.query(
        `INSERT INTO atomic50_catalog
           (mfr_sku,dw_sku,pattern_name,product_type,all_images,image_url,product_url,
            discontinued,on_shopify,last_scraped,created_at,updated_at)
         VALUES ($1,$2,$3,'Tin Ceiling Tile',$4::jsonb,$5,$6,false,false,now(),now(),now())`,
        [mfr, dw, s.title, imgsJson, firstImg, s.url]
      );
      inserted++; console.log(`  + NEW ${mfr} -> ${dw} (${s.url})`);
    }
  }

  // DISCO: individual-page rows absent from sitemap -> confirm with live 404 before flagging
  let discoFlagged = 0, missing = indivRows.filter((r) => !scrapedByUrl.has(normUrl(r.product_url)));
  const floorOk = scraped.length >= Math.floor(indivRows.length * 0.7);
  if (!DISCO_ON) {
    console.log(`  · disco DISABLED (DISCO=0); ${missing.length} individual rows not in sitemap this pass`);
  } else if (!floorOk) {
    console.log(`  ! disco SKIPPED — scraped ${scraped.length} < 70% of ${indivRows.length} individual rows (hiccup guard)`);
  } else {
    for (const r of missing) {
      const st = await statusOf(r.product_url);
      if (st === 404 || st === 410) {
        await db.query(`UPDATE atomic50_catalog SET discontinued=true, updated_at=now()
                          WHERE dw_sku=$1 AND discontinued IS DISTINCT FROM true`, [r.dw_sku]);
        discoFlagged++; console.log(`  - DISCO ${r.mfr_sku} (${r.product_url} -> ${st})`);
      } else {
        console.log(`  · ${r.mfr_sku} not in sitemap but url still ${st} — left as-is`);
      }
      await sleep(120);
    }
  }

  await db.end();
  const secs = ((Date.now() - t0) / 1000).toFixed(1);
  console.log(`\nDONE (${secs}s, $0 local): scraped=${scraped.length} inserted=${inserted} updated=${updated} disco_flagged=${discoFlagged}`);
}
main().catch((e) => { console.error('FATAL', e); process.exit(1); });