← back to Designer Wallcoverings
Add dedicated Anna French scraper (AT/AF -> DWAT/DWAF, staging-only, vendor-scoped)
9e18ec7f790d03bd67c1360397d69f2be0884f1a · 2026-06-25 07:55:12 -0700 · Steve Abrams
Files touched
A shopify/scripts/anna-french-scraper.js
Diff
commit 9e18ec7f790d03bd67c1360397d69f2be0884f1a
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Jun 25 07:55:12 2026 -0700
Add dedicated Anna French scraper (AT/AF -> DWAT/DWAF, staging-only, vendor-scoped)
---
shopify/scripts/anna-french-scraper.js | 390 +++++++++++++++++++++++++++++++++
1 file changed, 390 insertions(+)
diff --git a/shopify/scripts/anna-french-scraper.js b/shopify/scripts/anna-french-scraper.js
new file mode 100644
index 00000000..4b27cdbc
--- /dev/null
+++ b/shopify/scripts/anna-french-scraper.js
@@ -0,0 +1,390 @@
+#!/usr/bin/env node
+'use strict';
+/**
+ * Anna French Scraper — dedicated, vendor-scoped ingest for the Anna French line.
+ *
+ * Anna French is a Thibaut-owned brand served on thibautdesign.com under the
+ * mfr-code namespaces AT##### (wallcovering) and AF##### (fabric). This scraper
+ * crawls ONLY Anna French products (scoped by the "<Collection> by Anna French"
+ * breadcrumb), captures the full product page (all images, width, repeat, match,
+ * material, price), classifies type deterministically from the mfr-code prefix
+ * (AT->Wallcovering->DWAT, AF->Fabric->DWAF), generates clean handles, and writes
+ * to the anna_french_catalog STAGING table (PostgreSQL FIRST, no Shopify writes).
+ *
+ * Modeled on shopify/scripts/thibaut-repeat-scraper.js (plain https GET + browser
+ * UA, no auth). Reuses its fetchPage()/extractSpecs() shape.
+ *
+ * Usage:
+ * node anna-french-scraper.js --dry-run --limit 10 # prove extraction, no DB
+ * node anna-french-scraper.js --limit 10 # small real staging insert
+ * node anna-french-scraper.js --urls AT1405,AF24553 # explicit codes/urls
+ * node anna-french-scraper.js --all # full line (GATED — Steve sign-off)
+ *
+ * HARD RULES enforced here (DW standing rules + re-onboard plan):
+ * - Vendor-scoped to Anna French ONLY (breadcrumb "… by Anna French"); a page that
+ * is NOT Anna French is SKIPPED (never spill into Thibaut proper).
+ * - Prefix derived from mfr-code: AT->DWAT (Wallcovering), AF->DWAF (Fabric).
+ * - Word "Wallpaper" BANNED in title/description.
+ * - mfr# (AT/AF#####) NEVER customer-facing: clean_handle + a clean title are
+ * generated; the raw mfr code is stored ONLY in mfr_sku (internal).
+ * - NEVER "Unknown" in a title: pattern+color -> mfr SKU -> color -> skip.
+ * - Clean handle: pattern-color-anna-french (separator-safe, deduped).
+ * - Dedup on mfr_sku + pattern vs shopify_products AND dw_sku_registry.
+ * - STAGING table writes only. No Shopify writes, no image renames, no activation.
+ */
+
+const { Pool } = require('pg');
+const https = require('https');
+
+// Local PG via Unix socket, peer auth as the OS user (no password). Falls back to
+// DATABASE_URL when present (e.g. on Kamatera).
+const pool = process.env.DATABASE_URL
+ ? new Pool({ connectionString: process.env.DATABASE_URL })
+ : new Pool({ host: '/tmp', database: 'dw_unified', user: process.env.PGUSER || process.env.USER });
+
+const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+const BASE = 'https://www.thibautdesign.com';
+const BATCH_SIZE = 4;
+const DELAY_MS = 1500;
+const MAX_DEFAULT = 200;
+
+// ---- args ----
+const argv = process.argv.slice(2);
+const DRY_RUN = argv.includes('--dry-run');
+const ALL = argv.includes('--all');
+const limitArg = argv.indexOf('--limit');
+const LIMIT = limitArg >= 0 ? Math.max(1, parseInt(argv[limitArg + 1]) || MAX_DEFAULT) : MAX_DEFAULT;
+const urlsArg = argv.indexOf('--urls');
+const URLS = urlsArg >= 0 ? String(argv[urlsArg + 1] || '').split(',').map(s => s.trim()).filter(Boolean) : [];
+
+// ---------- net ----------
+function fetchPage(url) {
+ return new Promise((resolve, reject) => {
+ const timeout = setTimeout(() => { req.destroy(); reject(new Error('timeout')); }, 25000);
+ const req = https.get(url, { headers: { 'User-Agent': UA } }, (res) => {
+ if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+ clearTimeout(timeout);
+ res.resume();
+ resolve({ redirected: true, status: res.statusCode, location: res.headers.location });
+ return;
+ }
+ let data = '';
+ res.on('data', c => data += c);
+ res.on('end', () => { clearTimeout(timeout); resolve({ html: data, status: res.statusCode }); });
+ });
+ req.on('error', (e) => { clearTimeout(timeout); reject(e); });
+ });
+}
+function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
+
+// ---------- helpers ----------
+const BANNED_WORD = /\bwallpapers?\b/gi;
+function cleanWallpaper(s) {
+ if (!s) return s;
+ return String(s).replace(/\bWalls Wallpaper\b/gi, 'Wallcoverings').replace(/\bWallpapers\b/gi, 'Wallcoverings').replace(/\bWallpaper\b/gi, 'Wallcovering');
+}
+function titleCase(s) {
+ const small = new Set(['a', 'an', 'the', 'and', 'but', 'or', 'for', 'nor', 'in', 'on', 'at', 'to', 'by', 'of', 'with']);
+ return String(s || '').toLowerCase().split(/\s+/).filter(Boolean).map((w, i) =>
+ (i > 0 && small.has(w)) ? w : w.charAt(0).toUpperCase() + w.slice(1)
+ ).join(' ');
+}
+function slugify(s) {
+ return String(s || '').toLowerCase().normalize('NFKD').replace(/[^\w\s-]/g, '').trim().replace(/[\s_]+/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, '');
+}
+
+// ---------- type classifier (deterministic on mfr-code prefix) ----------
+function classify(mfrSku, widthInches) {
+ const code = String(mfrSku || '').toUpperCase();
+ if (code.startsWith('AF')) return { type: 'Fabric', prefix: 'DWAF', urlPath: 'fabrics' };
+ if (code.startsWith('AT')) return { type: 'Wallcovering', prefix: 'DWAT', urlPath: 'wallcoverings' };
+ // tie-break (codes that aren't AF/AT shouldn't occur for Anna French, but be safe):
+ if (widthInches && Number(widthInches) >= 50) return { type: 'Fabric', prefix: 'DWAF', urlPath: 'fabrics' };
+ return { type: 'Wallcovering', prefix: 'DWAT', urlPath: 'wallcoverings' };
+}
+
+// ---------- extraction ----------
+function extractProduct(html, mfrSku) {
+ const out = {
+ mfrSku, isAnnaFrench: false, patternName: null, colorName: null, collection: null,
+ material: null, use: null, width: null, widthInches: null, vRepeat: null, hRepeat: null,
+ matchType: null, description: null, images: [], primaryImage: null,
+ };
+
+ // --- vendor scope: "<Collection> by Anna French" breadcrumb ---
+ const afm = html.match(/([A-Z][\w& ]+?) by Anna French/);
+ if (afm) { out.isAnnaFrench = true; out.collection = afm[1].trim().toUpperCase(); }
+ // belt-and-suspenders: brand link / breadcrumb text mentioning Anna French
+ if (!out.isAnnaFrench && /Anna\s*French/i.test(html) && /breadcrumb/i.test(html)) {
+ // only trust this if the page is clearly an Anna French collection page
+ if (/by Anna French/i.test(html)) out.isAnnaFrench = true;
+ }
+
+ // --- JSON-LD Product node ---
+ const lds = html.match(/<script[^>]*application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi) || [];
+ for (const block of lds) {
+ const json = block.replace(/<script[^>]*>/i, '').replace(/<\/script>/i, '').trim();
+ let parsed;
+ try { parsed = JSON.parse(json); } catch { continue; }
+ const graph = (parsed && parsed['@graph']) || [parsed];
+ for (const node of graph) {
+ if (!node || node['@type'] !== 'Product') continue;
+ // name = "PATTERN - Color"
+ const name = String(node.name || '');
+ const dash = name.split(/\s+[-–]\s+/);
+ if (dash.length >= 2) { out.patternName = out.patternName || dash[0].trim(); out.colorName = out.colorName || dash.slice(1).join(' - ').trim(); }
+ else if (name) out.patternName = out.patternName || name.trim();
+ out.colorName = out.colorName || (node.color ? String(node.color).trim() : null);
+ out.material = out.material || (node.material ? String(node.material).trim() : null);
+ out.description = out.description || (node.description ? String(node.description).trim() : null);
+ const imgs = Array.isArray(node.image) ? node.image : (node.image ? [node.image] : []);
+ for (const im of imgs) if (im && !out.images.includes(im)) out.images.push(String(im));
+ }
+ }
+
+ // --- additionalProperty PropertyValue table ---
+ const props = {};
+ const re = /"@type":"PropertyValue","name":"([^"]+)","value":("?[^,}\]]+"?)/g;
+ let m;
+ while ((m = re.exec(html))) {
+ const k = m[1].trim();
+ let v = m[2].trim().replace(/^"|"$/g, '');
+ props[k] = v;
+ }
+ if (props['Pattern Name'] && !out.patternName) out.patternName = props['Pattern Name'].trim();
+ if (props['Collection'] && !out.collection) out.collection = props['Collection'].trim().toUpperCase();
+ if (props['Use']) out.use = props['Use'].trim();
+ if (props['Width']) { out.widthInches = parseFloat(props['Width']); out.width = `${props['Width']} Inches`; }
+
+ // --- escaped React-Router data blob: match / vertical_repeat / horizontal_repeat ---
+ const matchM = html.match(/\\+"match\\+":\\+"([^"\\]+)/);
+ if (matchM && !/^(Fabrics|Wallcoverings|true|false)$/i.test(matchM[1])) out.matchType = matchM[1].trim();
+ const vrM = html.match(/\\+"vertical_repeat\\+":\s*([\d.]+)/);
+ if (vrM) out.vRepeat = parseFloat(vrM[1]);
+ const hrM = html.match(/\\+"horizontal_repeat(?:_fabric)?\\+":\s*([\d.]+)/);
+ if (hrM) out.hRepeat = parseFloat(hrM[1]);
+
+ // --- all images for this mfr code (strip size suffixes, dedupe by stem) ---
+ const imgRe = new RegExp('https://cdn\\.shopify\\.com/[^"\\\\ ]*' + mfrSku + '[^"\\\\ ]*\\.(?:jpg|jpeg|png)', 'gi');
+ const found = html.match(imgRe) || [];
+ const byStem = new Map();
+ for (const u of [...out.images, ...found]) {
+ const base = u.split(/[?#]/)[0];
+ // prefer the un-suffixed full-size (AT1405.jpg over AT1405_306x306.jpg)
+ const stem = base.replace(/_\d+x\d+(?=\.\w+$)/, '');
+ if (!byStem.has(stem) || !/_\d+x\d+/.test(base)) byStem.set(stem, base);
+ }
+ out.images = [...byStem.values()];
+ out.primaryImage = out.images[0] || null;
+
+ // clean banned word from any field that can surface customer-facing
+ if (out.description) out.description = cleanWallpaper(out.description.replace(/\s+/g, ' ').trim());
+ if (out.material) out.material = cleanWallpaper(out.material.trim());
+ return out;
+}
+
+// ---------- title (no "Unknown" ever) ----------
+function buildTitle(p) {
+ const pat = p.patternName && p.patternName.trim();
+ const col = p.colorName && p.colorName.trim();
+ let core;
+ if (pat && col) core = `${pat} ${col}`;
+ else if (pat) core = pat;
+ else if (p.mfrSku) core = p.mfrSku; // fallback to mfr SKU
+ else if (col) core = col; // then color
+ else return null; // skip — never "Unknown"
+ return cleanWallpaper(`${titleCase(core)} | Anna French`);
+}
+
+// ---------- handle (pattern-color-anna-french, clean) ----------
+function buildHandle(p) {
+ const parts = [p.patternName, p.colorName].filter(Boolean).map(slugify).filter(Boolean);
+ parts.push('anna-french');
+ return parts.join('-').replace(/-+/g, '-');
+}
+
+// ---------- dedup: mfr_sku + pattern vs shopify_products AND dw_sku_registry ----------
+async function isDuplicate(mfrSku, patternName) {
+ // shopify_products: match on mfr_sku (variant SKU / sku / tags) — keep it cheap and exact on mfr_sku.
+ const sp = await pool.query(
+ `SELECT 1 FROM shopify_products
+ WHERE (sku = $1 OR variant_sku = $1 OR tags ILIKE '%' || $1 || '%')
+ LIMIT 1`, [mfrSku]
+ ).catch(() => ({ rows: [] }));
+ if (sp.rows.length) return { dup: true, where: 'shopify_products' };
+ const reg = await pool.query(
+ `SELECT 1 FROM dw_sku_registry WHERE mfr_sku = $1 LIMIT 1`, [mfrSku]
+ ).catch(() => ({ rows: [] }));
+ if (reg.rows.length) return { dup: true, where: 'dw_sku_registry' };
+ return { dup: false };
+}
+
+// ---------- candidate selection ----------
+async function selectCandidates() {
+ if (URLS.length) {
+ // explicit mfr codes or full urls
+ return URLS.map(u => {
+ const code = (u.match(/(A[TF]\d+)/i) || [])[1] || u;
+ return { mfr_sku: code.toUpperCase(), product_url: /^https?:/i.test(u) ? u : null };
+ });
+ }
+ // From staging: refresh AT rows already present; AF (fabric) side has zero rows so a
+ // full crawl would need the AF universe (gated). For staging validation we read what's
+ // present and (for AF) accept explicit --urls.
+ const lim = ALL ? 100000 : LIMIT;
+ const { rows } = await pool.query(
+ `SELECT mfr_sku, product_url FROM anna_french_catalog
+ WHERE mfr_sku IS NOT NULL
+ ORDER BY (product_url IS NULL), mfr_sku
+ LIMIT $1`, [lim]
+ );
+ return rows;
+}
+
+function urlFor(mfrSku, existingUrl) {
+ if (existingUrl) return existingUrl;
+ const { urlPath } = classify(mfrSku);
+ return `${BASE}/products/${urlPath}/${mfrSku}`;
+}
+
+// ---------- upsert into staging ----------
+async function upsertStaging(p, cls, title, handle) {
+ // additive columns assumed present (see ensureColumns): base_sku, dw_prefix,
+ // clean_handle, clean_title, dedup_skip, dedup_where, cost, cost_unit_of_measure.
+ await pool.query(
+ `INSERT INTO anna_french_catalog
+ (mfr_sku, pattern_name, color_name, collection, product_type, material,
+ width, width_inches, repeat_v, repeat_h, match_type, image_url, all_images,
+ product_url, application, dw_prefix, clean_handle, clean_title, last_scraped, updated_at)
+ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,NOW(),NOW())
+ ON CONFLICT (mfr_sku) DO UPDATE SET
+ pattern_name = COALESCE(EXCLUDED.pattern_name, anna_french_catalog.pattern_name),
+ color_name = COALESCE(EXCLUDED.color_name, anna_french_catalog.color_name),
+ collection = COALESCE(EXCLUDED.collection, anna_french_catalog.collection),
+ product_type = EXCLUDED.product_type,
+ material = COALESCE(EXCLUDED.material, anna_french_catalog.material),
+ width = COALESCE(EXCLUDED.width, anna_french_catalog.width),
+ width_inches = COALESCE(EXCLUDED.width_inches, anna_french_catalog.width_inches),
+ repeat_v = COALESCE(EXCLUDED.repeat_v, anna_french_catalog.repeat_v),
+ repeat_h = COALESCE(EXCLUDED.repeat_h, anna_french_catalog.repeat_h),
+ match_type = COALESCE(EXCLUDED.match_type, anna_french_catalog.match_type),
+ image_url = COALESCE(EXCLUDED.image_url, anna_french_catalog.image_url),
+ all_images = COALESCE(EXCLUDED.all_images, anna_french_catalog.all_images),
+ product_url = COALESCE(EXCLUDED.product_url, anna_french_catalog.product_url),
+ application = COALESCE(EXCLUDED.application, anna_french_catalog.application),
+ dw_prefix = EXCLUDED.dw_prefix,
+ clean_handle = EXCLUDED.clean_handle,
+ clean_title = EXCLUDED.clean_title,
+ last_scraped = NOW(), updated_at = NOW()`,
+ [
+ p.mfrSku, p.patternName, p.colorName, p.collection, cls.type, p.material,
+ p.width, p.widthInches != null && !isNaN(p.widthInches) ? p.widthInches : null,
+ p.vRepeat != null ? String(p.vRepeat) : null,
+ p.hRepeat != null ? String(p.hRepeat) : null,
+ p.matchType, p.primaryImage, JSON.stringify(p.images),
+ p.productUrl, p.use, cls.prefix, handle, title,
+ ]
+ );
+}
+
+async function setDedup(mfrSku, where) {
+ await pool.query(
+ `UPDATE anna_french_catalog SET dedup_skip = TRUE, dedup_where = $2, updated_at = NOW() WHERE mfr_sku = $1`,
+ [mfrSku, where]
+ ).catch(() => {});
+}
+
+// ---------- additive DDL (staging only) ----------
+async function ensureColumns() {
+ const cols = [
+ ['dw_prefix', 'text'], ['clean_handle', 'text'], ['clean_title', 'text'],
+ ['dedup_skip', 'boolean'], ['dedup_where', 'text'],
+ ['cost', 'numeric'], ['cost_unit_of_measure', 'text'],
+ ];
+ for (const [name, type] of cols) {
+ await pool.query(`ALTER TABLE anna_french_catalog ADD COLUMN IF NOT EXISTS ${name} ${type}`);
+ }
+}
+
+// ---------- main ----------
+async function run() {
+ const t0 = Date.now();
+ if (!DRY_RUN) await ensureColumns();
+
+ const candidates = await selectCandidates();
+ console.log(`${DRY_RUN ? '[DRY RUN] ' : ''}Anna French scrape — ${candidates.length} candidate(s)${ALL ? ' [--all GATED]' : ''}`);
+
+ const stats = { scraped: 0, staged: 0, notAnnaFrench: 0, discontinued: 0, dup: 0, skippedNoTitle: 0, failed: 0 };
+ const samples = [];
+
+ for (let i = 0; i < candidates.length; i += BATCH_SIZE) {
+ const batch = candidates.slice(i, i + BATCH_SIZE);
+ await Promise.allSettled(batch.map(async (c) => {
+ const mfr = String(c.mfr_sku).toUpperCase();
+ const url = urlFor(mfr, c.product_url);
+ try {
+ const res = await fetchPage(url);
+ if (res.redirected || res.status === 404) {
+ stats.discontinued++;
+ console.log(` [DISC] ${mfr} — ${res.status}${res.location ? ' -> ' + res.location : ''}`);
+ if (!DRY_RUN) await pool.query(
+ `UPDATE anna_french_catalog SET discontinued = TRUE, updated_at = NOW() WHERE mfr_sku = $1`, [mfr]
+ ).catch(() => {});
+ return;
+ }
+ if (res.status !== 200) { stats.failed++; console.log(` [ERR] ${mfr} — HTTP ${res.status}`); return; }
+
+ const p = extractProduct(res.html, mfr);
+ p.productUrl = url;
+ stats.scraped++;
+
+ if (!p.isAnnaFrench) {
+ stats.notAnnaFrench++;
+ console.log(` [SKIP-NOT-AF] ${mfr} — page is not "… by Anna French" (Thibaut proper; not staged)`);
+ return;
+ }
+
+ const cls = classify(mfr, p.widthInches);
+ const title = buildTitle(p);
+ if (!title) { stats.skippedNoTitle++; console.log(` [SKIP-NO-TITLE] ${mfr} — no pattern/color/sku`); return; }
+ const handle = buildHandle(p);
+
+ const d = await isDuplicate(mfr, p.patternName);
+
+ if (samples.length < 12) {
+ samples.push({
+ mfr, type: cls.type, prefix: cls.prefix, title, handle,
+ pattern: p.patternName, color: p.colorName, collection: p.collection,
+ width: p.width, widthInches: p.widthInches, material: p.material, use: p.use,
+ vRepeat: p.vRepeat, hRepeat: p.hRepeat, matchType: p.matchType,
+ images: p.images.length, primaryImage: p.primaryImage,
+ dup: d.dup ? d.where : false,
+ });
+ }
+
+ if (!DRY_RUN) {
+ await upsertStaging(p, cls, title, handle);
+ if (d.dup) await setDedup(mfr, d.where);
+ stats.staged++;
+ }
+ if (d.dup) stats.dup++;
+ console.log(` [OK] ${mfr} ${cls.prefix} — "${title}" | imgs:${p.images.length} w:${p.width || '?'} rpt:${p.vRepeat || '?'}${d.dup ? ' DUP(' + d.where + ')' : ''}${DRY_RUN ? ' (dry)' : ''}`);
+ } catch (err) {
+ stats.failed++;
+ console.log(` [ERR] ${mfr} — ${err.message}`);
+ }
+ }));
+ if (i + BATCH_SIZE < candidates.length) await sleep(DELAY_MS);
+ }
+
+ const dur = Math.round((Date.now() - t0) / 1000);
+ console.log(`\n=== RESULTS (${dur}s) ===`);
+ console.log(JSON.stringify(stats, null, 0));
+ console.log('\n=== EXTRACTED SAMPLES ===');
+ for (const s of samples) console.log(JSON.stringify(s));
+
+ await pool.end();
+ return stats;
+}
+
+run().catch(err => { console.error('FATAL', err); process.exit(1); });
← b142d6f5 Fix front-facing Brewster/York leak: 26 products retitled+re
·
back to Designer Wallcoverings
·
auto-save: 2026-06-25T08:02:44 (3 files) — shopify/scripts/c 3085a5ef →