← back to Patternbank Archive

scripts/ingest-pdp.js

274 lines

#!/usr/bin/env node
/**
 * Ingest phase: pull PDPs from crawl_queue, extract metadata, upsert.
 *
 * Usage:
 *   node scripts/ingest-pdp.js --limit=10
 *   node scripts/ingest-pdp.js --limit=100 --concurrency=2
 */
require('dotenv').config({ path: require('path').join(__dirname, '..', '.env') });
const cheerio = require('cheerio');
const pLimit = require('p-limit').default || require('p-limit');
const { withPage, sleep, UA } = require('../src/browser');
const db = require('../src/db');

const RATE = Number(process.env.RATE_LIMIT_MS || 2500);

function args() {
  const a = {};
  for (const arg of process.argv.slice(2)) {
    const m = arg.match(/^--([^=]+)=(.+)$/);
    if (m) a[m[1]] = m[2];
  }
  return a;
}

function patternIdFromUrl(url) {
  // PDP: /<designer>/designs/<numeric-id>-<slug>
  const m = url.match(/\/([A-Za-z0-9_-]+)\/designs\/(\d+)-([a-z0-9-]+)/);
  if (m) return m[2]; // numeric id is the stable key
  return url.replace(/^https?:\/\/[^/]+/, '');
}

function designerSlugFromUrl(url) {
  const m = url.match(/^https?:\/\/[^/]+\/([A-Za-z0-9_-]+)\/designs\//);
  return m ? m[1] : null;
}

function slugFromUrl(url) {
  const m = url.match(/\/designs\/(\d+-[a-z0-9-]+)/);
  return m ? m[1] : url;
}

function extractColors($) {
  const out = new Set();
  // Look for any element with inline background-color hex / rgb
  $('[style*="background"]').each((_, el) => {
    const s = $(el).attr('style') || '';
    const hex = s.match(/#([0-9a-fA-F]{6})/);
    if (hex) out.add('#' + hex[1].toLowerCase());
    const rgb = s.match(/rgb\((\d+),\s*(\d+),\s*(\d+)\)/);
    if (rgb) {
      const h = '#' + [rgb[1],rgb[2],rgb[3]].map(n => Number(n).toString(16).padStart(2,'0')).join('');
      out.add(h.toLowerCase());
    }
  });
  // Common swatch class patterns
  $('.swatch, [class*="swatch"], [class*="color-chip"]').each((_, el) => {
    const s = ($(el).attr('style')||'') + ' ' + ($(el).attr('data-color')||'');
    const hex = s.match(/#([0-9a-fA-F]{6})/);
    if (hex) out.add('#' + hex[1].toLowerCase());
  });
  return [...out].slice(0, 32);
}

function parsePdp(html, url) {
  const $ = cheerio.load(html);
  const pattern_id = patternIdFromUrl(url);
  const slug_full = slugFromUrl(url);
  const designer_from_url = designerSlugFromUrl(url);

  // JSON-LD first
  let jsonld = null;
  $('script[type="application/ld+json"]').each((_, el) => {
    try {
      const j = JSON.parse($(el).contents().text());
      if (Array.isArray(j)) j.forEach(x => { if (!jsonld && (x['@type']==='Product' || x.name)) jsonld = x; });
      else if (j['@type']==='Product' || j.name) jsonld = j;
    } catch {}
  });

  const og = (k) => $(`meta[property="og:${k}"]`).attr('content') || $(`meta[name="og:${k}"]`).attr('content');
  const title = (jsonld && jsonld.name)
    || og('title')
    || $('h1').first().text().trim()
    || $('title').text().trim();

  let designer_display_name = (jsonld && (jsonld.author?.name || jsonld.brand?.name)) || null;
  let designer_slug = designer_from_url;
  if (designer_slug && !designer_display_name) {
    // Find an anchor pointing at the designer profile root
    $(`a[href="/${designer_slug}"]`).each((_, el) => {
      const t = $(el).text().trim();
      if (t && t.length < 80 && !designer_display_name) designer_display_name = t;
    });
  }

  const description = (jsonld && jsonld.description) || og('description') || $('meta[name=description]').attr('content') || null;

  let price_usd = null;
  if (jsonld && jsonld.offers) {
    const o = Array.isArray(jsonld.offers) ? jsonld.offers[0] : jsonld.offers;
    if (o && o.price) price_usd = Number(o.price);
  }

  // Tags / breadcrumb category
  const tags = new Set();
  $('a[href*="/tag/"], a[href*="/tags/"], a[rel="tag"], .tag, [class*="tag-"]').each((_, el) => {
    const t = $(el).text().trim();
    if (t && t.length < 40) tags.add(t.toLowerCase());
  });
  let category = null, subcategory = null;
  const crumbs = [];
  $('[class*="breadcrumb"] a, nav.breadcrumb a, .breadcrumbs a').each((_, el) => {
    crumbs.push($(el).text().trim());
  });
  if (crumbs.length >= 2) category = crumbs[1] || null;
  if (crumbs.length >= 3) subcategory = crumbs[2] || null;

  // Images: og:image + main product image + gallery
  const images = new Set();
  const ogimg = og('image');
  if (ogimg) images.add(ogimg);
  $('img').each((_, el) => {
    const src = $(el).attr('src') || $(el).attr('data-src') || $(el).attr('data-original');
    if (!src) return;
    let abs = src;
    if (src.startsWith('//')) abs = 'https:' + src;
    else if (src.startsWith('/')) abs = 'https://patternbank.com' + src;
    if (!abs.startsWith('http')) return;
    // Filter out obvious icons / sprites
    if (/\b(icon|sprite|logo|avatar|favicon|emoji)\b/i.test(abs)) return;
    if (abs.endsWith('.svg')) return;
    images.add(abs);
  });

  const colors_hex = extractColors($);
  const license_label = $('[class*="license"]').first().text().trim() || null;
  const is_seamless = /seamless|repeat/i.test(html);

  return {
    pattern_id,
    slug: slug_full,
    title: (title||'').slice(0, 500),
    designer_slug,
    designer_display_name,
    source_url: url,
    category,
    subcategory,
    tags: [...tags],
    colors_hex,
    description: description ? description.slice(0, 4000) : null,
    license_label,
    price_usd,
    is_seamless,
    images: [...images],
    raw: { jsonld, og_title: og('title'), og_image: ogimg, crumbs },
  };
}

async function upsertPattern(p, runId) {
  if (p.designer_slug) {
    await db.q(
      `INSERT INTO designers (designer_slug, display_name)
       VALUES ($1,$2)
       ON CONFLICT (designer_slug) DO UPDATE
         SET display_name = COALESCE(EXCLUDED.display_name, designers.display_name),
             updated_at = NOW()`,
      [p.designer_slug, p.designer_display_name]
    );
  }
  await db.q(
    `INSERT INTO patterns (
        pattern_id, slug, title, designer_slug, designer_display_name,
        source_url, category, subcategory, tags, colors_hex, description,
        license_label, price_usd, is_seamless, raw_payload, ingest_run_id
      ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)
     ON CONFLICT (pattern_id) DO UPDATE SET
        title = EXCLUDED.title,
        designer_slug = COALESCE(EXCLUDED.designer_slug, patterns.designer_slug),
        designer_display_name = COALESCE(EXCLUDED.designer_display_name, patterns.designer_display_name),
        category = COALESCE(EXCLUDED.category, patterns.category),
        subcategory = COALESCE(EXCLUDED.subcategory, patterns.subcategory),
        tags = EXCLUDED.tags,
        colors_hex = EXCLUDED.colors_hex,
        description = COALESCE(EXCLUDED.description, patterns.description),
        license_label = COALESCE(EXCLUDED.license_label, patterns.license_label),
        price_usd = COALESCE(EXCLUDED.price_usd, patterns.price_usd),
        is_seamless = COALESCE(EXCLUDED.is_seamless, patterns.is_seamless),
        raw_payload = EXCLUDED.raw_payload,
        updated_at = NOW(),
        ingest_run_id = EXCLUDED.ingest_run_id`,
    [p.pattern_id, p.slug, p.title, p.designer_slug, p.designer_display_name,
     p.source_url, p.category, p.subcategory, p.tags, p.colors_hex, p.description,
     p.license_label, p.price_usd, p.is_seamless, p.raw, runId]
  );
  let imgN = 0;
  for (let i = 0; i < p.images.length; i++) {
    const u = p.images[i];
    // Avoid double-insert of same remote_url for same pattern_id
    const r = await db.q(
      `SELECT 1 FROM pattern_images WHERE pattern_id=$1 AND remote_url=$2`,
      [p.pattern_id, u]
    );
    if (r.rows.length) continue;
    await db.q(
      `INSERT INTO pattern_images (pattern_id, position, remote_url) VALUES ($1,$2,$3)`,
      [p.pattern_id, i, u]
    );
    imgN++;
  }
  return imgN;
}

async function ingestOne(url, runId) {
  let html = '', status = 0;
  try {
    const r = await withPage(async (page) => {
      const resp = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 60000 });
      return { status: resp ? resp.status() : 0, html: await page.content() };
    });
    status = r.status; html = r.html;
  } catch (e) {
    console.warn(`[pdp] err ${url}: ${e.message}`);
    await db.markQueue(url, 0, e.message.slice(0,200));
    return { ok: false };
  }
  if (status >= 400) {
    await db.markQueue(url, status, `HTTP ${status}`);
    return { ok: false };
  }
  try {
    const p = parsePdp(html, url);
    const imgN = await upsertPattern(p, runId);
    await db.markQueue(url, status, null);
    console.log(`[pdp] ✓ ${p.pattern_id} — "${(p.title||'').slice(0,50)}" — ${p.images.length} img (${imgN} new)`);
    return { ok: true, imagesNew: imgN };
  } catch (e) {
    console.warn(`[pdp] parse err ${url}: ${e.message}`);
    await db.markQueue(url, status, 'parse: '+e.message.slice(0,180));
    return { ok: false };
  }
}

(async () => {
  const a = args();
  const limit = Number(a.limit || 10);
  const concurrency = Math.max(1, Math.min(3, Number(a.concurrency || 2)));
  const runId = await db.startRun('pdp', `limit=${limit} concurrency=${concurrency}`);
  console.log(`[pdp] run=${runId} limit=${limit} concurrency=${concurrency} UA=${UA}`);
  const urls = await db.popQueue('pdp', limit);
  if (!urls.length) {
    console.log(`[pdp] queue empty`);
    await db.finishRun(runId, {});
    return db.pool.end();
  }
  console.log(`[pdp] popped ${urls.length} urls`);
  const lim = pLimit(concurrency);
  let upserted = 0, imagesUp = 0, errors = 0;
  await Promise.all(urls.map((u, i) => lim(async () => {
    if (i > 0) await sleep(RATE / concurrency);
    const r = await ingestOne(u, runId);
    if (r.ok) { upserted++; imagesUp += (r.imagesNew||0); } else { errors++; }
  })));
  await db.finishRun(runId, {
    pages_fetched: urls.length,
    patterns_seen: urls.length,
    patterns_upserted: upserted,
    images_upserted: imagesUp,
    errors,
  });
  console.log(`[pdp] done patterns=${upserted} new_imgs=${imagesUp} errors=${errors}`);
  await db.pool.end();
})().catch(e => { console.error(e); process.exit(1); });