← back to Dw Vendor Scrapers Local
recrawl-coordonne.js
244 lines
#!/usr/bin/env node
// ============================================================================
// Coordonne ADDITIVE all_images re-crawl — Codename: Carmen (full-page pass)
// ============================================================================
// WHY: the original scrape-coordonne.js (WordPress REST API) captured only a
// single hero image (yoast og_image[0]) and NO gallery / per-colorway swatch
// images. Steve's standing rule "always pull all data and images" requires
// every product to carry its FULL image set (all angles + colorway swatches)
// before it is import-ready. This driver re-crawls the actual product PAGE for
// every coordonne row that is missing all_images and fills all_images + specs.
//
// SAFETY / ADDITIVE-ONLY:
// * It NEVER inserts new rows. It only UPDATEs all_images + specs on rows that
// already exist (matched by the mfr_sku carried out of the DB).
// * It only touches rows whose all_images is empty AND whose product_url is a
// real coordonne.com page (the legacy already-on-Shopify cohort has no
// coordonne.com URL and is skipped — those are dedup'd out at publish anyway).
// * It does NOT push to Shopify, does NOT publish, does NOT change SKUs.
//
// MODES:
// --canary N re-crawl only the first N rows (default 5) — run this FIRST to
// confirm the gallery selector matches live HTML before the full run.
// --all re-crawl every eligible row (the full ~2,857-page pass).
// --net-new restrict to rows whose mfr_sku is NOT already in shopify_products
// (skip the legacy dup cohort entirely).
//
// Mirrors the harness of the FIXED recrawl-osborne.js (httpGet + worker pool).
// ============================================================================
require('dotenv').config({ path: '/root/DW-Agents/dw-price-stock/.env' });
const https = require('https');
const { pool } = require('/root/DW-Agents/vendor-scrapers/scraper-utils');
const TABLE = 'coordonne_catalog';
const SITE_BASE = 'https://coordonne.com';
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
const argv = process.argv.slice(2);
const FULL = argv.includes('--all');
const NET_NEW_ONLY = argv.includes('--net-new');
const DRY_RUN = argv.includes('--dry-run'); // extract + print only, NEVER writes
const canaryIdx = argv.indexOf('--canary');
const CANARY_N = canaryIdx >= 0 ? (parseInt(argv[canaryIdx + 1], 10) || 5) : (FULL ? 0 : 5);
function httpGet(url, maxRedirects = 4) {
return new Promise((resolve, reject) => {
const req = https.get(url, { timeout: 25000, headers: { 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml', 'Accept-Language': 'en-US,en;q=0.9' } }, (res) => {
if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location && maxRedirects > 0) {
const loc = res.headers.location.startsWith('http') ? res.headers.location : SITE_BASE + res.headers.location;
res.resume();
return httpGet(loc, maxRedirects - 1).then(resolve).catch(reject);
}
if (res.statusCode !== 200) { res.resume(); return reject(new Error('HTTP ' + res.statusCode)); }
let d = ''; res.on('data', c => d += c); res.on('end', () => resolve(d)); res.on('error', reject);
});
req.on('error', reject);
req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
});
}
const sleep = (ms) => new Promise(r => setTimeout(r, ms + Math.floor(Math.random() * 250)));
// Normalize a WP uploads URL to its full-size original by stripping the
// -<W>x<H> size suffix WooCommerce/WP appends to resized derivatives.
function toFullSize(u) {
return u.replace(/-\d+x\d+(\.(?:jpe?g|png|webp))(\?.*)?$/i, '$1');
}
// Junk filters — never store logos, icons, payment badges, flags, avatars,
// placeholders, or sprite assets as product images.
// `qualities` = Coordonne_Qualities_NonWoven/Vinyl/etc. — a generic material-spec
// badge shared identically across every SKU of that material; not a product photo,
// and storing it would violate the no-shared-image-across-SKUs rule.
const JUNK = /(logo|icon|favicon|placeholder|sprite|payment|visa|mastercard|paypal|flag|avatar|loader|spinner|swatch-sprite|woocommerce-placeholder|qualities)/i;
// Extract every product image from a coordonne product page.
// Covers WooCommerce-style galleries (data-large_image / gallery anchors) and
// any wp-content/uploads <img> on the page, normalized to full size + deduped.
function extractImagesFromHtml(html) {
const found = [];
const push = (raw) => {
if (!raw) return;
let u = raw.trim().replace(/&/g, '&');
if (u.startsWith('//')) u = 'https:' + u;
if (!/^https?:\/\//i.test(u)) return;
// Host-anchored: the URL must ITSELF be a coordonne.com uploads file, not a
// 3rd-party share link (pinterest/facebook) that merely carries a coordonne
// image in a ?media= / ?url= query param — those contain "wp-content/uploads"
// + ".jpg" as substrings and would otherwise slip through.
if (!/^https?:\/\/(www\.)?coordonne\.com\/wp-content\/uploads\//i.test(u)) return;
if (!/\.(jpe?g|png|webp)(\?.*)?$/i.test(u)) return; // real image files only
if (JUNK.test(u)) return;
u = toFullSize(u);
if (!found.includes(u)) found.push(u);
};
// 1) WooCommerce gallery: data-large_image="FULL.jpg" (the canonical full-res)
let m;
const reLarge = /data-large_image=["']([^"']+)["']/gi;
while ((m = reLarge.exec(html))) push(m[1]);
// 2) Gallery anchors wrapping the full image: <a href="...uploads...jpg">
const reAnchor = /<a[^>]+href=["']([^"']+wp-content\/uploads\/[^"']+\.(?:jpe?g|png|webp)[^"']*)["']/gi;
while ((m = reAnchor.exec(html))) push(m[1]);
// 3) Any <img src / data-src / srcset> pointing at uploads (gallery thumbs,
// colorway swatches, lifestyle shots) — normalized to full size + deduped.
const reImg = /<img\b[^>]*?\b(?:data-src|data-large_image|src)=["']([^"']+)["'][^>]*>/gi;
while ((m = reImg.exec(html))) push(m[1]);
const reSrcset = /\bsrcset=["']([^"']+)["']/gi;
while ((m = reSrcset.exec(html))) {
for (const part of m[1].split(',')) push(part.trim().split(/\s+/)[0]);
}
// 4) og:image as a fallback hero if the gallery yielded nothing.
const og = html.match(/<meta[^>]+property=["']og:image["'][^>]+content=["']([^"']+)["']/i);
if (og) push(og[1]);
return found;
}
const stripTags = (s) => String(s == null ? '' : s).replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim();
// Coordonne exposes real product specs in the WooCommerce variation JSON
// (data-product_variations), one entry per colorway. The visible page has NO
// woocommerce-product-attributes table — a naive whole-page fallback would
// scrape the cookie-consent table as "specs". So we parse the variation JSON,
// pick the variation whose sku matches THIS row's mfr_sku (falling back to the
// first variation for product-/theme-level fields), and map its real fields.
function extractSpecsFromHtml(html, mfrSku) {
const specs = {};
const m = html.match(/data-product_variations=(['"])([\s\S]*?)\1/i);
if (m) {
const json = m[2]
.replace(/"/g, '"').replace(/'/g, "'").replace(/"/g, '"')
.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
let vars = [];
try { vars = JSON.parse(json); } catch (_) { vars = []; }
if (Array.isArray(vars) && vars.length) {
const want = String(mfrSku || '').trim().toLowerCase();
const v = vars.find(x => String(x.sku || '').trim().toLowerCase() === want) || vars[0];
const add = (k, val) => {
const s = stripTags(val);
if (s && !/^(n\/?a|-|—|none)$/i.test(s)) specs[k] = s; // skip empty/"N/A" noise
};
const attrs = v.attributes || {};
add('composition', attrs.attribute_pa_composition || v.theme_composition);
add('care', v.theme_care);
add('roll_width', v._variable_roll_width_field);
add('roll_length', v._variable_roll_length_field);
add('panel_width', v._variable_panel_width_field);
add('ean', v.ean);
add('weight', v.weight_html || v.weight);
add('dimensions', v.dimensions_html);
for (const [ak, av] of Object.entries(attrs)) {
if (ak === 'attribute_pa_composition') continue;
add(ak.replace(/^attribute_(pa_)?/, ''), av);
}
}
}
// Secondary: a genuine product-attributes table if one is ever present.
// Strictly scoped to that class so the cookie-consent table is never matched.
const tableMatch = html.match(/<table[^>]*class=["'][^"']*woocommerce-product-attributes[^"']*["'][\s\S]*?<\/table>/i);
if (tableMatch) {
const reRow = /<tr[^>]*>[\s\S]*?<th[^>]*>([\s\S]*?)<\/th>[\s\S]*?<td[^>]*>([\s\S]*?)<\/td>[\s\S]*?<\/tr>/gi;
let r;
while ((r = reRow.exec(tableMatch[0]))) {
const k = stripTags(r[1]); const val = stripTags(r[2]);
if (k && val && k.length < 60 && !specs[k]) specs[k] = val;
}
}
return specs;
}
(async () => {
// Eligible rows: missing all_images + a real coordonne.com product page.
// Eligible = missing all_images OR contaminated with a 3rd-party share link
// (self-heals the 20 rows the pre-fix canary wrote with pinterest URLs).
let sql = `
SELECT mfr_sku, split_part(product_url,'?',1) AS url
FROM ${TABLE}
WHERE (all_images IS NULL OR all_images = '' OR all_images = '[]'
OR all_images ILIKE '%pinterest%' OR all_images ILIKE '%facebook.com%')
AND product_url ~ 'coordonne\\.com'
`;
if (NET_NEW_ONLY) {
sql += ` AND NOT EXISTS (SELECT 1 FROM shopify_products s WHERE upper(s.mfr_sku)=upper(${TABLE}.mfr_sku))`;
}
sql += ` ORDER BY mfr_sku`;
if (CANARY_N > 0) sql += ` LIMIT ${CANARY_N}`;
// Ensure target columns exist (idempotent).
await pool.query(`ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS all_images TEXT`);
await pool.query(`ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS specs JSONB`);
const { rows } = await pool.query(sql);
console.log(`Coordonne additive all_images re-crawl — ${rows.length} pages${CANARY_N > 0 ? ` (CANARY/limit ${CANARY_N})` : ' (FULL)'}${NET_NEW_ONLY ? ' [net-new only]' : ''}`);
let pages = 0, updated = 0, withImgs = 0, withSpecs = 0, noImgs = 0, errs = 0;
const concurrency = 3;
let idx = 0;
async function worker() {
while (idx < rows.length) {
const i = idx++;
const { mfr_sku, url } = rows[i];
try {
const html = await httpGet(url);
const images = extractImagesFromHtml(html);
const specs = extractSpecsFromHtml(html, mfr_sku);
pages++;
if (!images.length) { noImgs++; await sleep(300); continue; }
// ADDITIVE UPDATE — fill all_images (+ keep image_url as hero) + specs.
// Canary and --dry-run extract + print only; they NEVER write to prod.
if (!DRY_RUN && CANARY_N === 0) {
await pool.query(
`UPDATE ${TABLE}
SET all_images = $1,
image_url = COALESCE(NULLIF(image_url,''), $2),
specs = COALESCE(specs, '{}'::jsonb) || $3::jsonb,
last_scraped = NOW()
WHERE mfr_sku = $4`,
[JSON.stringify(images), images[0], JSON.stringify(specs), mfr_sku]
);
}
updated++; withImgs++;
if (Object.keys(specs).length) withSpecs++;
if (CANARY_N > 0) {
console.log(` ${mfr_sku}: ${images.length} imgs, ${Object.keys(specs).length} specs`);
console.log(` imgs: ${images.slice(0, 3).join('\n ')}${images.length > 3 ? '\n …' : ''}`);
console.log(` specs: ${JSON.stringify(specs)}`);
} else if (pages % 50 === 0) {
console.log(` ${pages}/${rows.length} | updated ${updated} | w/imgs ${withImgs} | w/specs ${withSpecs} | noImgs ${noImgs} | errs ${errs}`);
}
await sleep(350);
} catch (e) {
errs++;
if (errs % 10 === 0 || CANARY_N > 0) console.error(` err ${mfr_sku} ${url}: ${e.message}`);
await sleep(800);
}
}
}
await Promise.all(Array.from({ length: concurrency }, worker));
console.log(`\nDONE. pages=${pages} updated=${updated} with_imgs=${withImgs} with_specs=${withSpecs} no_imgs=${noImgs} errors=${errs}`);
await pool.end();
})().catch(e => { console.error('FATAL', e); pool.end().catch(() => {}); process.exit(1); });