`momentum-crawl.mjs`

88 lines
// Momentum authenticated catalog crawler (READ-ONLY). Authorized by Steve (trade creds).
// Drives the site's OWN numbered pager (the grid loads each page via its own search calls);
// harvests every product-color card: slug (join key), color number, pattern, price + unit,
// stock. Writes momentum-crawl.jsonl (one row per color number). No DB/Shopify writes.
import { chromium } from 'playwright';
import fs from 'node:fs';

const STATE = new URL('momentum-auth.json', import.meta.url).pathname;
const OUT   = new URL('momentum-crawl.jsonl', import.meta.url).pathname;
const BASE  = 'https://momentumco.com';
const CATS  = [ {c:2,name:'Wallcoverings'}, {c:7,name:'Acoustic'}, {c:3,name:'Wall Protection'},
                {c:1,name:'Upholstery'}, {c:6,name:'Drapery'} ];
if (!fs.existsSync(STATE)) { console.error('No auth state'); process.exit(1); }

const browser = await chromium.launch({ channel: 'chrome', headless: true });
const ctx = await browser.newContext({ storageState: STATE, viewport: { width: 1500, height: 1300 } });
const page = await ctx.newPage();
const all = new Map();
const out = fs.createWriteStream(OUT, { flags: 'w' });

const harvest = () => page.evaluate(() => {
  const recs = [];
  document.querySelectorAll('[data-testid="product-card"]').forEach(card => {
    const href = card.querySelector('a[href*="/products/"]')?.getAttribute('href') || '';
    const colornum = href.split('/products/')[1]?.split(/[?#]/)[0] || null;
    if (!colornum) return;
    const img = card.querySelector('img[src*="hi_res/"], source[srcset*="hi_res/"]');
    const s = img?.getAttribute('src') || img?.getAttribute('srcset') || '';
    const slug = (s.match(/hi_res\/([a-z0-9_]+-[a-z0-9_]+)\.jpg/i) || [])[1]?.toLowerCase() || null;
    const pattern = card.querySelector('img[alt]')?.getAttribute('alt')?.trim() || null;
    const text = (card.innerText || '').replace(/\s+/g, ' ').trim();
    const pm = text.match(/\$\s?([\d,]+\.\d{2})\s*(USD)?\s*\/\s*([A-Za-z.]+)/i);
    return recs.push({ colornum, slug, pattern,
      price: pm ? parseFloat(pm[1].replace(/,/g,'')) : null,
      unit: pm ? pm[3].toUpperCase().replace(/\./g,'') : null,
      stock: (text.match(/([\d.,]+)\s*(yards?|yd|rolls?|sf|sy)\s+in stock/i) || [])[0] || null,
      isNew: /\bNEW\b/.test(text),
      colorCount: (text.match(/(\d+)\s+Colors?/i) || [])[1] || null,
      text: text.slice(0,150) });
  });
  return recs;
});
const firstCard = () => page.evaluate(() =>
  document.querySelector('[data-testid="product-card"] a[href*="/products/"]')?.getAttribute('href') || null);
const take = (recs, cat) => { for (const r of recs) if (r.colornum && !all.has(r.colornum)) {
  r.category = cat; all.set(r.colornum, r); out.write(JSON.stringify(r) + '\n'); } };

for (const cat of CATS) {
  console.log(`\n=== ${cat.name} (c=${cat.c}) ===`);
  await page.goto(`${BASE}/products?c=${cat.c}&forceSync=1`, { waitUntil: 'domcontentloaded', timeout: 60000 }).catch(()=>{});
  await page.waitForTimeout(5000);
  const total = await page.evaluate(() => parseInt((document.body.innerText.match(/([\d,]+)\s+products/i)||[])[1]?.replace(/,/g,'')||'0',10));
  const maxPage = await page.evaluate(() => Math.max(1, ...[...document.querySelectorAll('button[aria-label^="Go to page"]')]
    .map(b => parseInt((b.getAttribute('aria-label').match(/page (\d+)/)||[])[1]||'0',10))));
  console.log(`  total=${total} pages=${maxPage}`);
  const before = all.size;
  take(await harvest(), cat.name);
  let pg = 1;
  while (pg < maxPage + 3) {
    const prevFirst = await firstCard();
    const next = page.locator('button[aria-label="Go to next page"]');
    const disabled = await next.first().isDisabled().catch(()=>true);
    if (disabled) { console.log(`  next disabled at page ${pg}`); break; }
    await next.first().click({ timeout: 8000 }).catch(()=>{});
    // wait for the grid to swap (first card href changes)
    let swapped = false;
    for (let w = 0; w < 30; w++) { await page.waitForTimeout(300);
      if (await firstCard() !== prevFirst) { swapped = true; break; } }
    await page.waitForTimeout(500);
    take(await harvest(), cat.name);
    pg++;
    if (!swapped) { console.log(`  no swap after page ${pg}; stopping cat`); break; }
    if (pg % 10 === 0) console.log(`  page ${pg}/${maxPage} cat=${all.size-before}/${total} total=${all.size}`);
  }
  console.log(`  ${cat.name} done: +${all.size - before} (running ${all.size})`);
}
out.end();
const recs = [...all.values()];
fs.writeFileSync(new URL('momentum-crawl-summary.json', import.meta.url).pathname, JSON.stringify({
  totalColornums: all.size,
  withSlug: recs.filter(r=>r.slug).length,
  withPrice: recs.filter(r=>r.price!=null).length,
  units: recs.reduce((a,r)=>{a[r.unit||'?']=(a[r.unit||'?']||0)+1;return a;},{}),
  byCategory: recs.reduce((a,r)=>{a[r.category]=(a[r.category]||0)+1;return a;},{}),
}, null, 2));
console.log('\n=== DONE ===\n' + fs.readFileSync(new URL('momentum-crawl-summary.json', import.meta.url).pathname,'utf8'));
await browser.close();