`fetch-ol-images.cjs`

94 lines
'use strict';
// Final pass: through Browserbase (cloud IP can reach the blocked O&L host), for each PDP
// (a) scroll to trigger lazy-load, (b) read the magento gallery JSON (full colorway swatches),
// (c) fetch the chosen image bytes in-browser and return base64 so we can save locally.
const os = require('os'), fs = require('fs'), path = require('path');
require('dotenv').config({ path: os.homedir() + '/.claude/skills/browserbase/.env' });
const Browserbase = require('@browserbasehq/sdk').default;
const { chromium } = require('playwright-core');
const sleep = ms => new Promise(r => setTimeout(r, ms));
const OUT = '/Users/macstudio3/Projects/dw-marketing-reels/sourced-images/imageless-14';
fs.mkdirSync(OUT, { recursive: true });

// pages to open; each returns full gallery so we can pick colorway-correct swatch
const pages = [
  { key:'W7814-01', url:'https://www.osborneandlittle.com/colonnato-w7814.html' },
  { key:'W7682-01', url:'https://www.osborneandlittle.com/coralline-w7682.html' },
  { key:'W7024-03', url:'https://www.osborneandlittle.com/japanese-garden-w7024-w7024.html' },
  { key:'W6541-02', url:'https://www.osborneandlittle.com/peacock-w6541.html' },
  { key:'W5223-03', url:'https://www.osborneandlittle.com/tamba-w5223.html' },
  { key:'W7334-1',  url:'https://www.osborneandlittle.com/trailing-orchid-w7334.html' },
  { key:'EUR-70328',url:'https://www.osborneandlittle.com/audley_W6297-W6297.html' },
  { key:'EUR-70402',url:'https://www.osborneandlittle.com/canestra_W6341-W6341.html' },
  { key:'EUR-70351',url:'https://www.osborneandlittle.com/chameleon_W6305-W6305.html' },
  { key:'NCW4270',  url:'https://www.osborneandlittle.com/coromandel_NCW4270-NCW4270.html' },
];

async function galleryOf(page){
  // scroll to load lazy imgs
  await page.evaluate(async()=>{ for(let y=0;y<3000;y+=400){window.scrollTo(0,y);await new Promise(r=>setTimeout(r,150));} window.scrollTo(0,0); });
  await sleep(1500);
  return await page.evaluate(() => {
    const out = { imgs:[], magento:[] };
    document.querySelectorAll('img').forEach(im => {
      const s = im.currentSrc || im.src || im.getAttribute('data-src') || '';
      if (/media\/catalog\/product/i.test(s) && !/AVATAR|white_logo|brand\//i.test(s)) out.imgs.push(s);
    });
    // magento fotorama/gallery json
    const html = document.documentElement.innerHTML;
    const re = /"(?:img|full|thumb)":"(https?:[^"]*media\/catalog\/product[^"]+)"/g; let m;
    while((m=re.exec(html))) out.magento.push(m[1].replace(/\\\//g,'/'));
    return { imgs:[...new Set(out.imgs)], magento:[...new Set(out.magento)] };
  });
}

async function fetchBytes(page, url){
  return await page.evaluate(async (u) => {
    const r = await fetch(u); if(!r.ok) return null;
    const b = await r.arrayBuffer(); const bytes = new Uint8Array(b);
    let bin=''; for(let i=0;i<bytes.length;i++) bin+=String.fromCharCode(bytes[i]);
    return { b64: btoa(bin), type: r.headers.get('content-type')||'', len: bytes.length };
  }, url);
}

(async () => {
  const bb = new Browserbase({ apiKey: process.env.BROWSERBASE_API_KEY });
  const session = await bb.sessions.create({ projectId: process.env.BROWSERBASE_PROJECT_ID, browserSettings: { solveCaptchas: true } });
  console.error('session', session.id);
  const browser = await chromium.connectOverCDP(session.connectUrl);
  const page = browser.contexts()[0].pages()[0] || await browser.contexts()[0].newPage();
  page.setDefaultTimeout(45000);
  const manifest = [];
  for (const p of pages){
    const rec = { key:p.key, url:p.url };
    try {
      await page.goto(p.url, { waitUntil:'domcontentloaded' }).catch(()=>{});
      await sleep(2500);
      rec.gallery = await galleryOf(page);
      // choose candidate imgs to download: SKU-named first, else all product imgs (non-hero)
      const all = [...new Set([...rec.gallery.magento, ...rec.gallery.imgs])];
      rec.candidates = all;
      // download every candidate that isn't the collection hero pattern (mpiowebpcache/bff0.. = og hero)
      const dl = all.filter(u=>/catalog\/product/i.test(u));
      rec.saved = [];
      for (let i=0;i<dl.length && i<8;i++){
        const u = dl[i];
        const r = await fetchBytes(page, u);
        if (r && r.len>2000){
          const ext = /webp/i.test(r.type)?'webp':(/png/i.test(r.type)?'png':'jpg');
          const base = (u.split('/').pop()||('img'+i)).replace(/\.(webp|jpg|jpeg|png).*$/i,'');
          const fn = `${p.key}__${base}.${ext}`;
          fs.writeFileSync(path.join(OUT, fn), Buffer.from(r.b64,'base64'));
          rec.saved.push({ file:fn, from:u, bytes:r.len, type:r.type });
        }
      }
    } catch(e){ rec.error = e.message; }
    console.error(`[${p.key}] mag=${rec.gallery?rec.gallery.magento.length:0} img=${rec.gallery?rec.gallery.imgs.length:0} saved=${rec.saved?rec.saved.length:0} ${rec.error||''}`);
    manifest.push(rec);
  }
  await browser.close().catch(()=>{});
  const fp = '/Users/macstudio3/Projects/dw-marketing-reels/fetch-ol-manifest.json';
  fs.writeFileSync(fp, JSON.stringify(manifest,null,2));
  console.log(fp);
})().catch(e=>{ console.error('FATAL', e.message); process.exit(1); });