`scripts/discover.js`

138 lines
#!/usr/bin/env node
/**
 * Discover phase: walk patternbank categories, enqueue PDP URLs into crawl_queue.
 *
 * Patternbank URL structure (confirmed 2026-05-13):
 *   - Top categories: /categories/<top-id-name>            e.g. /categories/1-womenswear
 *   - Subcategories:  /categories/<top>/<sub>              e.g. /categories/1-womenswear/10-floral
 *   - Pagination:     ?page=N&sort=recent&licence=all
 *   - PDP:            /<designer-slug>/designs/<id>-<slug>
 *
 * Usage:
 *   node scripts/discover.js                                   # discover top cats from root
 *   node scripts/discover.js --top=1-womenswear --max-pages=5
 *   node scripts/discover.js --cat=/categories/1-womenswear/10-floral --max-pages=10
 *   node scripts/discover.js --auto --max-pages=3              # walk every top × sub
 */
require('dotenv').config({ path: require('path').join(__dirname, '..', '.env') });
const cheerio = require('cheerio');
const { withPage, sleep, UA } = require('../src/browser');
const db = require('../src/db');

const RATE = Number(process.env.RATE_LIMIT_MS || 2500);
const ORIGIN = 'https://patternbank.com';

function args() {
  const a = {};
  for (const arg of process.argv.slice(2)) {
    const m = arg.match(/^--([^=]+)=(.+)$/);
    if (m) a[m[1]] = m[2];
    else if (arg.startsWith('--')) a[arg.slice(2)] = true;
  }
  return a;
}

const PDP_RE = /^\/([A-Za-z0-9_-]+)\/designs\/(\d+-[a-z0-9-]+)\/?$/;

function pdpLinks($) {
  const urls = new Set();
  $('a[href]').each((_, el) => {
    const href = ($(el).attr('href') || '').split('?')[0].split('#')[0];
    const m = href.match(PDP_RE);
    if (m) urls.add(ORIGIN + href);
  });
  return [...urls];
}

function categoryLinks($) {
  // /categories/<top> and /categories/<top>/<sub>
  const tops = new Set(), subs = new Set();
  $('a[href]').each((_, el) => {
    const href = ($(el).attr('href') || '').split('?')[0];
    const mTop = href.match(/^\/categories\/(\d+-[a-z0-9-]+)$/);
    if (mTop) tops.add(href);
    const mSub = href.match(/^\/categories\/(\d+-[a-z0-9-]+)\/(\d+-[a-z0-9-]+)$/);
    if (mSub) subs.add(href);
  });
  return { tops: [...tops].map(p => ORIGIN + p), subs: [...subs].map(p => ORIGIN + p) };
}

async function fetchAndParse(url) {
  return withPage(async (page) => {
    const resp = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
    // Patternbank renders product cards on initial HTML — short scroll to trigger any lazy
    await page.waitForTimeout(1500);
    for (let s = 0; s < 6; s++) {
      await page.evaluate(() => window.scrollBy(0, 2200));
      await page.waitForTimeout(400);
    }
    return { status: resp ? resp.status() : 0, html: await page.content(), finalUrl: page.url() };
  });
}

async function crawlListing(baseUrl, maxPages, runId) {
  const counters = { pages: 0, enqueued: 0, errors: 0 };
  for (let p = 1; p <= maxPages; p++) {
    const url = `${baseUrl}?licence=all&sort=recent&page=${p}`;
    console.log(`[discover] page ${p}/${maxPages}: ${url}`);
    let r;
    try { r = await fetchAndParse(url); }
    catch (e) {
      console.warn(`[discover] err ${e.message}`);
      counters.errors++; await db.markQueue(url, 0, e.message.slice(0,200)); break;
    }
    await db.markQueue(url, r.status, r.status>=400 ? `HTTP ${r.status}` : null);
    if (r.status >= 400) { counters.errors++; break; }
    const $ = cheerio.load(r.html);
    const pdps = pdpLinks($);
    for (const u of pdps) { await db.enqueue(u, 'pdp', 50); counters.enqueued++; }
    counters.pages++;
    console.log(`[discover]   page=${p} +${pdps.length} pdps (cumulative ${counters.enqueued})`);
    if (pdps.length === 0) break; // ran past end
    await sleep(RATE);
  }
  await db.finishRun(runId, {
    pages_fetched: counters.pages,
    patterns_seen: counters.enqueued,
    errors: counters.errors,
  });
  return counters;
}

async function discoverCategoryTree() {
  // Hit /categories/ root (or any category page) to discover all top × sub
  const r = await fetchAndParse(ORIGIN + '/categories/1-womenswear');
  const $ = cheerio.load(r.html);
  return categoryLinks($);
}

(async () => {
  const a = args();

  if (a.auto) {
    const maxPages = Number(a['max-pages'] || 3);
    console.log(`[discover] AUTO mode — walking every top×sub @ maxPages=${maxPages}`);
    const tree = await discoverCategoryTree();
    console.log(`[discover] discovered ${tree.tops.length} tops, ${tree.subs.length} subs`);
    const runId = await db.startRun('discover', `auto subs=${tree.subs.length} maxPages=${maxPages}`);
    let totalEnq = 0;
    for (const sub of tree.subs) {
      const c = await crawlListing(sub, maxPages, runId);
      totalEnq += c.enqueued;
      console.log(`[discover] sub=${sub} +${c.enqueued}`);
    }
    console.log(`[discover] AUTO done — totalEnqueued=${totalEnq}`);
    await db.pool.end(); return;
  }

  let start = a.cat
    || (a.top ? `${ORIGIN}/categories/${a.top}` : `${ORIGIN}/categories/1-womenswear/10-floral`);
  if (start.startsWith('/')) start = ORIGIN + start;
  const maxPages = Number(a['max-pages'] || 3);
  console.log(`[discover] start=${start} maxPages=${maxPages}`);
  const runId = await db.startRun('discover', `start=${start} maxPages=${maxPages}`);
  const counters = await crawlListing(start, maxPages, runId);
  console.log(`[discover] done`, counters);
  await db.pool.end();
})().catch(e => { console.error(e); process.exit(1); });