`reference/scrape-cowtan.js`

72 lines
#!/usr/bin/env node
// Cowtan & Tout Catalog Scraper - designs.cowtan.com AJAX API
// Target table: cowtan_tout_catalog
const https = require('https');
const { pool, upsertProduct, wait, closePool, getCatalogCount } = require('./scraper-utils');
const TABLE = 'cowtan_tout_catalog';
const API_BASE = 'https://designs.cowtan.com';
const IMG_BASE = 'https://d2mq91o692rj7w.cloudfront.net/images/Products';
const BRAND_MAP = { 1: 'Cowtan & Tout', 2: 'Colefax and Fowler', 3: 'Jane Churchill', 5: 'Larsen', 6: 'Manuel Canovas' };
const TYPE_MAP = { W: 'Wallcovering', F: 'Fabric', T: 'Trimming' };
let totalInserted = 0, totalErrors = 0;

function httpPost(url, body) {
  return new Promise(function(resolve, reject) {
    var parsed = new URL(url);
    var opts = { hostname: parsed.hostname, port: 443, path: parsed.pathname,
      method: 'POST', timeout: 30000,
      headers: { 'Content-Type': 'application/x-www-form-urlencoded', 'Content-Length': Buffer.byteLength(body),
        'User-Agent': 'Mozilla/5.0', 'Accept': 'application/json', 'X-Requested-With': 'XMLHttpRequest' } };
    var req = https.request(opts, function(res) {
      if (res.statusCode === 301 || res.statusCode === 302) {
        return httpPost(res.headers.location, body).then(resolve).catch(reject);
      }
      var data = '';
      res.on('data', function(c) { data += c; });
      res.on('end', function() { try { resolve(JSON.parse(data)); } catch(e) { reject(new Error('JSON: ' + e.message + ' (len=' + data.length + ')')); } });
      res.on('error', reject);
    });
    req.on('error', reject); req.on('timeout', function() { req.destroy(); reject(new Error('timeout')); });
    req.write(body); req.end();
  });
}

function buildBody(pi, tv) { var p = ['Keywords=','NewOnly=false']; ['F','W','T'].forEach(function(t,i) {
  p.push('Types%5B'+i+'%5D.Value='+t); p.push('Types%5B'+i+'%5D.Selected='+(t===tv?'true':'false')); });
  p.push('PageIndex='+pi); return p.join('&'); }

async function scrapeType(tv) { console.log('\n  Fetching: '+TYPE_MAP[tv]+'...'); var all = [];
  var f = await httpPost(API_BASE+'/Search', buildBody(0, tv)); var tot = f.TotalResults||0;
  console.log('  Total: '+tot); if (!f.Products||!f.Products.length) return all;
  var pp = f.Products[0]||[]; all.push.apply(all, pp); console.log('  Page 0: '+pp.length);
  var mx = Math.min(Math.ceil(tot/100), 200);
  for (var pi=1; pi<mx; pi++) { try { await wait(200,150);
    var pg = await httpPost(API_BASE+'/Search', buildBody(pi, tv));
    if (!pg.Products||!pg.Products.length) break; var it = pg.Products[0]||[]; if (!it.length) break;
    all.push.apply(all, it); if ((pi+1)%10===0) console.log('  Page '+pi+': '+all.length);
  } catch(e) { console.error('  Pg '+pi+': '+e.message); totalErrors++; await wait(2000); } }
  console.log('  '+TYPE_MAP[tv]+' total: '+all.length); return all; }

function mapProduct(r, tv) { if (!r.StockCode) return null;
  return { mfr_sku: r.StockCode, pattern_name: r.ProductName||null, color_name: r.ColourName||null,
    collection: r.PatternBook||null, material: BRAND_MAP[r.BrandCode]||'Cowtan & Tout',
    product_type: TYPE_MAP[tv], price_retail: r.SamplePrice||null,
    image_url: IMG_BASE+'/'+r.StockCode.replace(/\//g,'-')+'.jpg',
    product_url: 'https://designs.cowtan.com/Design/'+r.StockCode.replace(/\//g,'-'), in_stock: true }; }

async function main() { console.log('='.repeat(70)); console.log('  COWTAN & TOUT CATALOG SCRAPER'); console.log('='.repeat(70));
  var t0 = Date.now();
  await pool.query('CREATE TABLE IF NOT EXISTS '+TABLE+' (id SERIAL PRIMARY KEY, mfr_sku TEXT UNIQUE, dw_sku TEXT, pattern_name TEXT, color_name TEXT, collection TEXT, product_type TEXT, width TEXT, length TEXT, repeat_v TEXT, repeat_h TEXT, material TEXT, price_retail NUMERIC(10,2), price_trade NUMERIC(10,2), image_url TEXT, product_url TEXT, in_stock BOOLEAN DEFAULT TRUE, discontinued BOOLEAN DEFAULT FALSE, shopify_product_id TEXT, last_scraped TIMESTAMPTZ, created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW())');
  console.log('  Table ready.'); var wp = await scrapeType('W'); await wait(1000,500); var fp = await scrapeType('F');
  console.log('\n  Upserting...'); var ar = [];
  wp.forEach(function(r){var p=mapProduct(r,'W');if(p)ar.push(p);});
  fp.forEach(function(r){var p=mapProduct(r,'F');if(p)ar.push(p);});
  var s={},u=[]; ar.forEach(function(p){if(!s[p.mfr_sku]){s[p.mfr_sku]=true;u.push(p);}});
  console.log('  Unique: '+u.length);
  for(var i=0;i<u.length;i++){try{var id=await upsertProduct(TABLE,u[i]);if(id)totalInserted++;}catch(e){totalErrors++;}
    if((i+1)%500===0)console.log('  DB: '+(i+1)+'/'+u.length+' ins='+totalInserted);}
  var el=((Date.now()-t0)/1000).toFixed(1); var ct=await getCatalogCount(TABLE);
  console.log('\n'+'='.repeat(70)); console.log('  DONE: wall='+wp.length+' fab='+fp.length+' ins='+totalInserted+' err='+totalErrors+' db='+ct);
  console.log('  Time: '+el+'s'); console.log('='.repeat(70)); await closePool(); }
main().catch(function(e){console.error('FATAL:',e);closePool().then(function(){process.exit(1);});});