← back to Dw Vendor Scrapers Local
coordonne recrawl: reject 3rd-party share-link images (pinterest/fb), parse real specs from variation JSON, make canary/--dry-run non-writing + self-heal contaminated rows
0851c0c88571e5981dc006e1026be43d4951a30d · 2026-06-10 16:37:12 -0700 · SteveStudio2
Files touched
Diff
commit 0851c0c88571e5981dc006e1026be43d4951a30d
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date: Wed Jun 10 16:37:12 2026 -0700
coordonne recrawl: reject 3rd-party share-link images (pinterest/fb), parse real specs from variation JSON, make canary/--dry-run non-writing + self-heal contaminated rows
---
recrawl-coordonne.js | 91 +++++++++++++++++++++++++++++++++++++++-------------
1 file changed, 69 insertions(+), 22 deletions(-)
diff --git a/recrawl-coordonne.js b/recrawl-coordonne.js
index bc1bb24..db559bb 100644
--- a/recrawl-coordonne.js
+++ b/recrawl-coordonne.js
@@ -37,6 +37,7 @@ const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (
const argv = process.argv.slice(2);
const FULL = argv.includes('--all');
const NET_NEW_ONLY = argv.includes('--net-new');
+const DRY_RUN = argv.includes('--dry-run'); // extract + print only, NEVER writes
const canaryIdx = argv.indexOf('--canary');
const CANARY_N = canaryIdx >= 0 ? (parseInt(argv[canaryIdx + 1], 10) || 5) : (FULL ? 0 : 5);
@@ -77,7 +78,11 @@ function extractImagesFromHtml(html) {
let u = raw.trim().replace(/&/g, '&');
if (u.startsWith('//')) u = 'https:' + u;
if (!/^https?:\/\//i.test(u)) return;
- if (!/wp-content\/uploads\//i.test(u)) return; // product imagery only
+ // Host-anchored: the URL must ITSELF be a coordonne.com uploads file, not a
+ // 3rd-party share link (pinterest/facebook) that merely carries a coordonne
+ // image in a ?media= / ?url= query param — those contain "wp-content/uploads"
+ // + ".jpg" as substrings and would otherwise slip through.
+ if (!/^https?:\/\/(www\.)?coordonne\.com\/wp-content\/uploads\//i.test(u)) return;
if (!/\.(jpe?g|png|webp)(\?.*)?$/i.test(u)) return; // real image files only
if (JUNK.test(u)) return;
u = toFullSize(u);
@@ -109,27 +114,65 @@ function extractImagesFromHtml(html) {
return found;
}
-// Pull the WooCommerce additional-information / attributes table as a specs map.
-function extractSpecsFromHtml(html) {
+const stripTags = (s) => String(s == null ? '' : s).replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim();
+
+// Coordonne exposes real product specs in the WooCommerce variation JSON
+// (data-product_variations), one entry per colorway. The visible page has NO
+// woocommerce-product-attributes table — a naive whole-page fallback would
+// scrape the cookie-consent table as "specs". So we parse the variation JSON,
+// pick the variation whose sku matches THIS row's mfr_sku (falling back to the
+// first variation for product-/theme-level fields), and map its real fields.
+function extractSpecsFromHtml(html, mfrSku) {
const specs = {};
+ const m = html.match(/data-product_variations=(['"])([\s\S]*?)\1/i);
+ if (m) {
+ const json = m[2]
+ .replace(/"/g, '"').replace(/'/g, "'").replace(/"/g, '"')
+ .replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
+ let vars = [];
+ try { vars = JSON.parse(json); } catch (_) { vars = []; }
+ if (Array.isArray(vars) && vars.length) {
+ const want = String(mfrSku || '').trim().toLowerCase();
+ const v = vars.find(x => String(x.sku || '').trim().toLowerCase() === want) || vars[0];
+ const add = (k, val) => { const s = stripTags(val); if (s) specs[k] = s; };
+ const attrs = v.attributes || {};
+ add('composition', attrs.attribute_pa_composition || v.theme_composition);
+ add('care', v.theme_care);
+ add('roll_width', v._variable_roll_width_field);
+ add('roll_length', v._variable_roll_length_field);
+ add('panel_width', v._variable_panel_width_field);
+ add('ean', v.ean);
+ add('weight', v.weight_html || v.weight);
+ add('dimensions', v.dimensions_html);
+ for (const [ak, av] of Object.entries(attrs)) {
+ if (ak === 'attribute_pa_composition') continue;
+ add(ak.replace(/^attribute_(pa_)?/, ''), av);
+ }
+ }
+ }
+ // Secondary: a genuine product-attributes table if one is ever present.
+ // Strictly scoped to that class so the cookie-consent table is never matched.
const tableMatch = html.match(/<table[^>]*class=["'][^"']*woocommerce-product-attributes[^"']*["'][\s\S]*?<\/table>/i);
- const scope = tableMatch ? tableMatch[0] : html;
- const reRow = /<tr[^>]*>[\s\S]*?<th[^>]*>([\s\S]*?)<\/th>[\s\S]*?<td[^>]*>([\s\S]*?)<\/td>[\s\S]*?<\/tr>/gi;
- let m;
- while ((m = reRow.exec(scope))) {
- const k = m[1].replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim();
- const v = m[2].replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim();
- if (k && v && k.length < 60) specs[k] = v;
+ if (tableMatch) {
+ const reRow = /<tr[^>]*>[\s\S]*?<th[^>]*>([\s\S]*?)<\/th>[\s\S]*?<td[^>]*>([\s\S]*?)<\/td>[\s\S]*?<\/tr>/gi;
+ let r;
+ while ((r = reRow.exec(tableMatch[0]))) {
+ const k = stripTags(r[1]); const val = stripTags(r[2]);
+ if (k && val && k.length < 60 && !specs[k]) specs[k] = val;
+ }
}
return specs;
}
(async () => {
// Eligible rows: missing all_images + a real coordonne.com product page.
+ // Eligible = missing all_images OR contaminated with a 3rd-party share link
+ // (self-heals the 20 rows the pre-fix canary wrote with pinterest URLs).
let sql = `
SELECT mfr_sku, split_part(product_url,'?',1) AS url
FROM ${TABLE}
- WHERE (all_images IS NULL OR all_images = '' OR all_images = '[]')
+ WHERE (all_images IS NULL OR all_images = '' OR all_images = '[]'
+ OR all_images ILIKE '%pinterest%' OR all_images ILIKE '%facebook.com%')
AND product_url ~ 'coordonne\\.com'
`;
if (NET_NEW_ONLY) {
@@ -155,24 +198,28 @@ function extractSpecsFromHtml(html) {
try {
const html = await httpGet(url);
const images = extractImagesFromHtml(html);
- const specs = extractSpecsFromHtml(html);
+ const specs = extractSpecsFromHtml(html, mfr_sku);
pages++;
if (!images.length) { noImgs++; await sleep(300); continue; }
// ADDITIVE UPDATE — fill all_images (+ keep image_url as hero) + specs.
- await pool.query(
- `UPDATE ${TABLE}
- SET all_images = $1,
- image_url = COALESCE(NULLIF(image_url,''), $2),
- specs = COALESCE(specs, '{}'::jsonb) || $3::jsonb,
- last_scraped = NOW()
- WHERE mfr_sku = $4`,
- [JSON.stringify(images), images[0], JSON.stringify(specs), mfr_sku]
- );
+ // Canary and --dry-run extract + print only; they NEVER write to prod.
+ if (!DRY_RUN && CANARY_N === 0) {
+ await pool.query(
+ `UPDATE ${TABLE}
+ SET all_images = $1,
+ image_url = COALESCE(NULLIF(image_url,''), $2),
+ specs = COALESCE(specs, '{}'::jsonb) || $3::jsonb,
+ last_scraped = NOW()
+ WHERE mfr_sku = $4`,
+ [JSON.stringify(images), images[0], JSON.stringify(specs), mfr_sku]
+ );
+ }
updated++; withImgs++;
if (Object.keys(specs).length) withSpecs++;
if (CANARY_N > 0) {
console.log(` ${mfr_sku}: ${images.length} imgs, ${Object.keys(specs).length} specs`);
- console.log(` ${images.slice(0, 3).join('\n ')}${images.length > 3 ? '\n …' : ''}`);
+ console.log(` imgs: ${images.slice(0, 3).join('\n ')}${images.length > 3 ? '\n …' : ''}`);
+ console.log(` specs: ${JSON.stringify(specs)}`);
} else if (pages % 50 === 0) {
console.log(` ${pages}/${rows.length} | updated ${updated} | w/imgs ${withImgs} | w/specs ${withSpecs} | noImgs ${noImgs} | errs ${errs}`);
}
← 5c43b67 Add recrawl-schumacher.js: additive all_images completeness
·
back to Dw Vendor Scrapers Local
·
coordonne recrawl: drop shared Coordonne_Qualities material f432b38 →