`scripts/clothing-apis/calibrate-spa.js`

121 lines
#!/usr/bin/env node
/**
 * SPA selector calibrator.
 *
 * Usage:
 *   node scripts/clothing-apis/calibrate-spa.js <brand-id>
 *
 * Steps:
 *   1. Load the brand entry from brands.json (must exist under headless_spa[]).
 *   2. Open the products page via Browserbase, wait for network idle, pull
 *      the rendered HTML, cache to /tmp/spa-<id>.html.
 *   3. Probe a battery of common card selectors. Print the count for each.
 *   4. For the best-scoring card selector, probe likely title/price/image/link
 *      sub-selectors inside the first card.
 *   5. Print a ready-to-paste `selectors` block.
 *
 * No interactive prompt — pure batch mode so it can run from CI / cron.
 * Once selectors look good, paste them into brands.json + flip `disabled: false`.
 *
 * Idempotent: re-running uses the cached HTML if it's < 1 hour old.
 */
const fs = require('fs');
const path = require('path');
const { fetchRenderedHtml, available } = require('./headless-spa');

const BRANDS = JSON.parse(fs.readFileSync(path.join(__dirname, 'brands.json'), 'utf8'));

// Common card selectors across the e-commerce SPA ecosystem.
const CARD_SELECTORS = [
  '.product-card', '.product', '.ProductCard', '.product-tile', '.product-item',
  '[data-test-id="product-card"]', '[data-testid="product-card"]', '[data-test="product-card"]',
  '[data-product-card]', '[data-product]', 'article.product',
  '.grid__item--product', '.collection-product', '.collection-product-card',
  'li.product', '.product-grid-item', '.shop-item', '.shop-card',
];

// Within a card, candidate selectors per field.
const TITLE_SELECTORS = ['.product-card__title', '.product__title', 'h2', 'h3', '.title', '[data-test*="title"]', '.name', '.product-name'];
const PRICE_SELECTORS = ['.product-card__price', '.product__price', '.price', '[data-test*="price"]', '.amount', '.product-price'];
const IMAGE_SELECTORS = ['img'];
const LINK_SELECTORS = ['a'];

async function main() {
  const brandId = process.argv[2];
  if (!brandId) {
    console.error('usage: node scripts/clothing-apis/calibrate-spa.js <brand-id>');
    console.error('available: ' + (BRANDS.headless_spa || []).map(b => b.id).join(', '));
    process.exit(1);
  }
  const brand = (BRANDS.headless_spa || []).find(b => b.id === brandId);
  if (!brand) {
    console.error(`brand ${brandId} not in brands.json headless_spa`);
    process.exit(1);
  }
  if (!available()) {
    console.error('BROWSERBASE_API_KEY / BROWSERBASE_PROJECT_ID not set in env or skill .env');
    process.exit(1);
  }

  const cacheFile = `/tmp/spa-${brandId}.html`;
  const cacheAgeMs = fs.existsSync(cacheFile) ? Date.now() - fs.statSync(cacheFile).mtimeMs : Infinity;
  let html;
  if (cacheAgeMs < 60 * 60 * 1000) {
    console.log(`[cache hit] reusing ${cacheFile} (age ${Math.round(cacheAgeMs/1000)}s)`);
    html = fs.readFileSync(cacheFile, 'utf8');
  } else {
    const url = `https://${brand.domain}${brand.products_path || '/products'}`;
    console.log(`[fetch] ${url} via Browserbase (this takes ~10-30s)`);
    html = await fetchRenderedHtml(url, { waitMs: 5000 });
    fs.writeFileSync(cacheFile, html);
    console.log(`[cache] wrote ${cacheFile} (${(html.length / 1024).toFixed(0)} KB)`);
  }

  let cheerio;
  try { cheerio = require('cheerio'); }
  catch { console.error('cheerio not installed — run `npm i cheerio`'); process.exit(1); }
  const $ = cheerio.load(html);

  // Probe 1 — which card selector matches a sensible number of cards?
  console.log('\n=== card selector candidates ===');
  const cardScores = CARD_SELECTORS.map(s => ({ s, n: $(s).length }));
  cardScores.sort((a, b) => b.n - a.n);
  cardScores.slice(0, 8).forEach(c => console.log(`  ${String(c.n).padStart(4)} × ${c.s}`));
  const best = cardScores.find(c => c.n >= 4 && c.n <= 500) || cardScores[0];
  if (!best || best.n === 0) {
    console.log('\nNo card selector matched. The page may be lazy-rendered (try a longer waitMs in headless-spa.js fetchRenderedHtml) or use a fully custom DOM structure (open /tmp/spa-' + brandId + '.html and craft selectors by hand).');
    process.exit(2);
  }
  console.log(`\n[pick] card = "${best.s}" (${best.n} matches)`);

  // Probe 2 — inside the first card, which sub-selectors hit?
  const firstCard = $(best.s).first();
  function probe(name, candidates) {
    const hits = candidates.map(c => ({ c, txt: firstCard.find(c).first().text().trim().slice(0, 60), exists: firstCard.find(c).length > 0 })).filter(x => x.exists);
    console.log(`\n=== ${name} candidates inside first card ===`);
    hits.forEach(h => console.log(`  ${h.c.padEnd(35)} → ${JSON.stringify(h.txt)}`));
    return hits[0]?.c || candidates[0];
  }
  const titleSel = probe('title', TITLE_SELECTORS);
  const priceSel = probe('price', PRICE_SELECTORS);

  // image + link probes (different — they want attr not text)
  const imgEl = firstCard.find('img').first();
  const imgSrc = imgEl.attr('src') || imgEl.attr('data-src') || imgEl.attr('srcset')?.split(',')[0]?.trim().split(' ')[0];
  const linkEl = firstCard.find('a').first();
  const linkHref = linkEl.attr('href');
  console.log(`\n=== image inside first card ===`);
  console.log(`  ${imgSrc ? 'FOUND: ' + (imgSrc.slice(0, 90)) : 'no <img> with src/data-src/srcset'}`);
  console.log(`\n=== link inside first card ===`);
  console.log(`  ${linkHref ? 'FOUND: ' + linkHref.slice(0, 90) : 'no <a href>'}`);

  // Final paste-ready block
  console.log(`\n=== READY-TO-PASTE selectors for brands.json → headless_spa[${brandId}] ===`);
  console.log(JSON.stringify({
    selectors: { card: best.s, title: titleSel, price: priceSel, image: 'img', link: 'a' }
  }, null, 2));
  console.log(`\nIf the values above look right, flip "disabled": false in brands.json and run:\n  curl -sS -X POST -H 'content-type: application/json' -d '{"kind":"headless","id":"${brandId}"}' http://127.0.0.1:9777/api/admin/import-catalog`);
}

if (require.main === module) main().catch(e => { console.error(e); process.exit(1); });