← back to Designer Wallcoverings
Anna French scraper: full-page capture (all specs+cost+care+real description+all-images), no York/cost leak
c3b4e551e18861074437f49bc504af4b8288ab46 · 2026-06-25 08:26:03 -0700 · Steve
Files touched
M shopify/scripts/anna-french-scraper.js
Diff
commit c3b4e551e18861074437f49bc504af4b8288ab46
Author: Steve <steve@designerwallcoverings.com>
Date: Thu Jun 25 08:26:03 2026 -0700
Anna French scraper: full-page capture (all specs+cost+care+real description+all-images), no York/cost leak
---
shopify/scripts/anna-french-scraper.js | 162 ++++++++++++++++++++++++++-------
1 file changed, 130 insertions(+), 32 deletions(-)
diff --git a/shopify/scripts/anna-french-scraper.js b/shopify/scripts/anna-french-scraper.js
index 4b27cdbc..d25f2842 100644
--- a/shopify/scripts/anna-french-scraper.js
+++ b/shopify/scripts/anna-french-scraper.js
@@ -103,12 +103,40 @@ function classify(mfrSku, widthInches) {
return { type: 'Wallcovering', prefix: 'DWAT', urlPath: 'wallcoverings' };
}
+// ---------- full-page spec blob parser ----------
+// Thibaut-family product pages embed the FULL product record (every spec, cost,
+// care flag, collection) in a triple-backslash-escaped React-Router data blob:
+// \\\"vertical_repeat\\\":10.25,\\\"construction\\\":\\\"Non Woven\\\", ...
+// We parse the segment around the product spec keys into a flat {key: value} map.
+// INTERNAL-ONLY fields (sample_book_vendor = "York Wallcoverings", unit_cost, abc_code,
+// etc.) live here too — they are captured for cost but NEVER surfaced customer-facing.
+function parseSpecBlob(html) {
+ // anchor on a spec-region key that is present for both fabric and wallcovering
+ let idx = html.indexOf('\\"horizontal_repeat_fabric');
+ if (idx < 0) idx = html.indexOf('\\"vertical_repeat');
+ if (idx < 0) idx = html.indexOf('\\"construction');
+ if (idx < 0) return {};
+ const seg = html.slice(Math.max(0, idx - 4000), idx + 6000);
+ const re = /\\{3}"([a-z0-9_]+)\\{3}":\s*(\\{3}"[^"\\]*\\{3}"|-?[\d.]+|null|true|false)/gi;
+ const out = {};
+ let m;
+ while ((m = re.exec(seg))) {
+ let v = m[2];
+ if (v.startsWith('\\')) v = v.replace(/\\+"/g, '').trim(); // unwrap string value
+ out[m[1]] = v === 'null' ? null : v;
+ }
+ return out;
+}
+
// ---------- extraction ----------
function extractProduct(html, mfrSku) {
const out = {
mfrSku, isAnnaFrench: false, patternName: null, colorName: null, collection: null,
- material: null, use: null, width: null, widthInches: null, vRepeat: null, hRepeat: null,
- matchType: null, description: null, images: [], primaryImage: null,
+ material: null, use: null, width: null, widthInches: null, length: null,
+ vRepeat: null, hRepeat: null, matchType: null, finish: null, application: null,
+ coverage: null, weight: null, care: null, fireRating: null,
+ cost: null, costUnit: null, description: null, metaDescription: null,
+ images: [], primaryImage: null,
};
// --- vendor scope: "<Collection> by Anna French" breadcrumb ---
@@ -116,11 +144,10 @@ function extractProduct(html, mfrSku) {
if (afm) { out.isAnnaFrench = true; out.collection = afm[1].trim().toUpperCase(); }
// belt-and-suspenders: brand link / breadcrumb text mentioning Anna French
if (!out.isAnnaFrench && /Anna\s*French/i.test(html) && /breadcrumb/i.test(html)) {
- // only trust this if the page is clearly an Anna French collection page
if (/by Anna French/i.test(html)) out.isAnnaFrench = true;
}
- // --- JSON-LD Product node ---
+ // --- JSON-LD Product node (name, color, primary image, meta-grade description) ---
const lds = html.match(/<script[^>]*application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi) || [];
for (const block of lds) {
const json = block.replace(/<script[^>]*>/i, '').replace(/<\/script>/i, '').trim();
@@ -129,60 +156,115 @@ function extractProduct(html, mfrSku) {
const graph = (parsed && parsed['@graph']) || [parsed];
for (const node of graph) {
if (!node || node['@type'] !== 'Product') continue;
- // name = "PATTERN - Color"
const name = String(node.name || '');
const dash = name.split(/\s+[-–]\s+/);
if (dash.length >= 2) { out.patternName = out.patternName || dash[0].trim(); out.colorName = out.colorName || dash.slice(1).join(' - ').trim(); }
else if (name) out.patternName = out.patternName || name.trim();
out.colorName = out.colorName || (node.color ? String(node.color).trim() : null);
out.material = out.material || (node.material ? String(node.material).trim() : null);
- out.description = out.description || (node.description ? String(node.description).trim() : null);
const imgs = Array.isArray(node.image) ? node.image : (node.image ? [node.image] : []);
for (const im of imgs) if (im && !out.images.includes(im)) out.images.push(String(im));
}
}
- // --- additionalProperty PropertyValue table ---
+ // --- additionalProperty PropertyValue table (Pattern Name / Collection / Use / Width) ---
const props = {};
const re = /"@type":"PropertyValue","name":"([^"]+)","value":("?[^,}\]]+"?)/g;
let m;
while ((m = re.exec(html))) {
- const k = m[1].trim();
- let v = m[2].trim().replace(/^"|"$/g, '');
- props[k] = v;
+ props[m[1].trim()] = m[2].trim().replace(/^"|"$/g, '');
}
if (props['Pattern Name'] && !out.patternName) out.patternName = props['Pattern Name'].trim();
if (props['Collection'] && !out.collection) out.collection = props['Collection'].trim().toUpperCase();
- if (props['Use']) out.use = props['Use'].trim();
+ if (props['Use']) out.use = out.application = props['Use'].trim();
if (props['Width']) { out.widthInches = parseFloat(props['Width']); out.width = `${props['Width']} Inches`; }
- // --- escaped React-Router data blob: match / vertical_repeat / horizontal_repeat ---
- const matchM = html.match(/\\+"match\\+":\\+"([^"\\]+)/);
- if (matchM && !/^(Fabrics|Wallcoverings|true|false)$/i.test(matchM[1])) out.matchType = matchM[1].trim();
- const vrM = html.match(/\\+"vertical_repeat\\+":\s*([\d.]+)/);
- if (vrM) out.vRepeat = parseFloat(vrM[1]);
- const hrM = html.match(/\\+"horizontal_repeat(?:_fabric)?\\+":\s*([\d.]+)/);
- if (hrM) out.hRepeat = parseFloat(hrM[1]);
+ // --- FULL spec blob (the real source for material/repeat/length/weight/cost/care) ---
+ const b = parseSpecBlob(html);
+ const num = (v) => { const n = parseFloat(v); return isNaN(n) ? null : n; };
+ // material / content
+ if (b.construction) out.material = cleanWallpaper(String(b.construction).trim());
+ // collection (book)
+ if (b.book_collection && !out.collection) out.collection = String(b.book_collection).trim().toUpperCase();
+ if (b.associated_book && !out.collection) out.collection = String(b.associated_book).trim().toUpperCase();
+ // match type (skip generic category strings)
+ if (b.match && !/^(fabrics?|wallcoverings?|true|false|null)$/i.test(b.match)) out.matchType = String(b.match).trim();
+ // width (blob wins if JSON-LD lacked it)
+ if (out.widthInches == null && b.width != null) { const w = num(b.width); if (w) { out.widthInches = w; out.width = `${b.width} Inches`; } }
+ // repeats — fabric uses *_fabric keys, wallcovering uses bare keys
+ out.vRepeat = num(b.vertical_repeat) ?? num(b.vertical_repeat_fabric);
+ out.hRepeat = num(b.horizontal_repeat) ?? num(b.horizontal_repeat_fabric);
+ // length — single_roll_length (wallcovering, feet) ; fabric is cut-to-order (no length)
+ const len = num(b.single_roll_length) ?? num(b.SingleRollLength);
+ if (len && len > 0) out.length = `${len} Feet`;
+ // weight
+ if (b.unit_weight && num(b.unit_weight)) out.weight = `${b.unit_weight}${b.weight_units ? ' ' + String(b.weight_units).toUpperCase() : ' LB'}`;
+ // application / use category
+ if (!out.application && b.web_category_1) out.application = String(b.web_category_1).trim();
+ // coverage (wallcovering single roll sq ft)
+ const sqft = num(b.single_roll_square_foot);
+ if (sqft) out.coverage = `${sqft} sq ft per single roll`;
+ // care flags -> human string
+ const careBits = [];
+ if (b.strippable === '1' || b.strippable === 'true' || b.strippable === true) careBits.push('Strippable');
+ if (b.unpasted === '1' || b.unpasted === 'true' || b.unpasted === true) careBits.push('Unpasted');
+ if (b.pretrimmed === '1' || b.pretrimmed === 'true' || b.pretrimmed === true) careBits.push('Pretrimmed');
+ if (b.washable === '1' || b.washable === 'true' || b.washable === true) careBits.push('Washable');
+ if (careBits.length) out.care = careBits.join(', ');
+ // fire rating (any non-null flammability_*)
+ for (let i = 1; i <= 5; i++) { const fr = b[`flammability_${i}`]; if (fr && fr !== 'null' && !out.fireRating) out.fireRating = String(fr).trim(); }
+ // COST (internal) — unit_cost is wholesale; unit-of-measure S=Single Roll, Y=Yard
+ if (b.unit_cost != null && num(b.unit_cost)) {
+ out.cost = Math.round(num(b.unit_cost) * 100) / 100;
+ out.costUnit = b.u_m === 'Y' ? 'YARD' : (b.u_m === 'S' ? 'Single Roll' : null);
+ }
+
+ // --- meta description (the only real descriptive sentence Thibaut exposes) ---
+ const md = html.match(/<meta\s+name=["']description["']\s+content=["']([^"']+)["']/i);
+ if (md) out.metaDescription = cleanWallpaper(md[1].replace(/&/g, '&').replace(/\s+/g, ' ').trim());
- // --- all images for this mfr code (strip size suffixes, dedupe by stem) ---
- const imgRe = new RegExp('https://cdn\\.shopify\\.com/[^"\\\\ ]*' + mfrSku + '[^"\\\\ ]*\\.(?:jpg|jpeg|png)', 'gi');
+ // --- ALL genuine images for THIS mfr code only (siblings AT1441/AT1443 belong to
+ // OTHER SKUs — never attach them). Capture every same-code variant
+ // (AT1442.jpg, AT1442_2.jpg, AT1442_room.jpg, etc.), strip size suffixes,
+ // dedupe by stem, prefer the full-size un-suffixed file. ---
+ const imgRe = new RegExp('https://cdn\\.shopify\\.com/[^"\\\\ )]*' + mfrSku + '(?:_[a-z0-9]+)?(?:_\\d+x\\d+)?\\.(?:jpg|jpeg|png|webp)', 'gi');
const found = html.match(imgRe) || [];
const byStem = new Map();
for (const u of [...out.images, ...found]) {
const base = u.split(/[?#]/)[0];
- // prefer the un-suffixed full-size (AT1405.jpg over AT1405_306x306.jpg)
- const stem = base.replace(/_\d+x\d+(?=\.\w+$)/, '');
+ // only THIS code's images (defensive: the JSON-LD image is already this code)
+ const fname = base.split('/').pop();
+ if (!new RegExp('^' + mfrSku + '(?:[_.]|$)', 'i').test(fname)) continue;
+ const stem = base.replace(/_\d+x\d+(?=\.\w+$)/, ''); // collapse AT1442_306x306 -> AT1442
if (!byStem.has(stem) || !/_\d+x\d+/.test(base)) byStem.set(stem, base);
}
out.images = [...byStem.values()];
out.primaryImage = out.images[0] || null;
- // clean banned word from any field that can surface customer-facing
- if (out.description) out.description = cleanWallpaper(out.description.replace(/\s+/g, ' ').trim());
+ // --- build a clean, real product description (NEVER a placeholder, NEVER leak
+ // York/sample_book_vendor or unit_cost). Prefer the meta description; enrich
+ // with the composed spec sentence. interior-designer lexicon pass happens in
+ // the STEP-3 normalizer, not here. ---
+ out.description = buildDescription(out);
+
if (out.material) out.material = cleanWallpaper(out.material.trim());
return out;
}
+// ---------- description builder (clean, on-brand, no leaks, no placeholder) ----------
+function buildDescription(p) {
+ const parts = [];
+ if (p.metaDescription && p.metaDescription.length > 20) parts.push(p.metaDescription);
+ const specBits = [];
+ if (p.material) specBits.push(`Crafted from ${p.material.toLowerCase()}`);
+ if (p.width) specBits.push(`${p.width.toLowerCase()} wide`);
+ if (p.vRepeat) specBits.push(`with a ${p.vRepeat}" vertical repeat`);
+ if (p.matchType) specBits.push(`${p.matchType.toLowerCase()}`);
+ if (specBits.length) parts.push(specBits.join(', ').replace(/^./, c => c.toUpperCase()) + '.');
+ const txt = cleanWallpaper(parts.join(' ').replace(/\s+/g, ' ').trim());
+ return txt && txt.length >= 24 ? txt : null; // null -> gate flags Needs-Description
+}
+
// ---------- title (no "Unknown" ever) ----------
function buildTitle(p) {
const pat = p.patternName && p.patternName.trim();
@@ -254,9 +336,11 @@ async function upsertStaging(p, cls, title, handle) {
await pool.query(
`INSERT INTO anna_french_catalog
(mfr_sku, pattern_name, color_name, collection, product_type, material,
- width, width_inches, repeat_v, repeat_h, match_type, image_url, all_images,
- product_url, application, dw_prefix, clean_handle, clean_title, last_scraped, updated_at)
- VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,NOW(),NOW())
+ width, width_inches, length, repeat_v, repeat_h, match_type, finish,
+ application, coverage, fire_rating, ai_description, image_url, all_images,
+ product_url, dw_prefix, clean_handle, clean_title, cost, cost_unit_of_measure,
+ last_scraped, updated_at)
+ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,NOW(),NOW())
ON CONFLICT (mfr_sku) DO UPDATE SET
pattern_name = COALESCE(EXCLUDED.pattern_name, anna_french_catalog.pattern_name),
color_name = COALESCE(EXCLUDED.color_name, anna_french_catalog.color_name),
@@ -265,24 +349,34 @@ async function upsertStaging(p, cls, title, handle) {
material = COALESCE(EXCLUDED.material, anna_french_catalog.material),
width = COALESCE(EXCLUDED.width, anna_french_catalog.width),
width_inches = COALESCE(EXCLUDED.width_inches, anna_french_catalog.width_inches),
+ length = COALESCE(EXCLUDED.length, anna_french_catalog.length),
repeat_v = COALESCE(EXCLUDED.repeat_v, anna_french_catalog.repeat_v),
repeat_h = COALESCE(EXCLUDED.repeat_h, anna_french_catalog.repeat_h),
match_type = COALESCE(EXCLUDED.match_type, anna_french_catalog.match_type),
+ finish = COALESCE(EXCLUDED.finish, anna_french_catalog.finish),
+ application = COALESCE(EXCLUDED.application, anna_french_catalog.application),
+ coverage = COALESCE(EXCLUDED.coverage, anna_french_catalog.coverage),
+ fire_rating = COALESCE(EXCLUDED.fire_rating, anna_french_catalog.fire_rating),
+ ai_description = COALESCE(EXCLUDED.ai_description, anna_french_catalog.ai_description),
image_url = COALESCE(EXCLUDED.image_url, anna_french_catalog.image_url),
all_images = COALESCE(EXCLUDED.all_images, anna_french_catalog.all_images),
product_url = COALESCE(EXCLUDED.product_url, anna_french_catalog.product_url),
- application = COALESCE(EXCLUDED.application, anna_french_catalog.application),
dw_prefix = EXCLUDED.dw_prefix,
clean_handle = EXCLUDED.clean_handle,
clean_title = EXCLUDED.clean_title,
+ cost = COALESCE(EXCLUDED.cost, anna_french_catalog.cost),
+ cost_unit_of_measure = COALESCE(EXCLUDED.cost_unit_of_measure, anna_french_catalog.cost_unit_of_measure),
last_scraped = NOW(), updated_at = NOW()`,
[
p.mfrSku, p.patternName, p.colorName, p.collection, cls.type, p.material,
p.width, p.widthInches != null && !isNaN(p.widthInches) ? p.widthInches : null,
+ p.length,
p.vRepeat != null ? String(p.vRepeat) : null,
p.hRepeat != null ? String(p.hRepeat) : null,
- p.matchType, p.primaryImage, JSON.stringify(p.images),
- p.productUrl, p.use, cls.prefix, handle, title,
+ p.matchType, p.finish, p.application, p.coverage, p.fireRating,
+ p.description, p.primaryImage, JSON.stringify(p.images),
+ p.productUrl, cls.prefix, handle, title,
+ p.cost, p.costUnit,
]
);
}
@@ -355,9 +449,13 @@ async function run() {
samples.push({
mfr, type: cls.type, prefix: cls.prefix, title, handle,
pattern: p.patternName, color: p.colorName, collection: p.collection,
- width: p.width, widthInches: p.widthInches, material: p.material, use: p.use,
+ width: p.width, widthInches: p.widthInches, length: p.length,
+ material: p.material, application: p.application,
vRepeat: p.vRepeat, hRepeat: p.hRepeat, matchType: p.matchType,
- images: p.images.length, primaryImage: p.primaryImage,
+ weight: p.weight, care: p.care, coverage: p.coverage, fireRating: p.fireRating,
+ cost: p.cost, costUnit: p.costUnit,
+ images: p.images.length, allImages: p.images,
+ description: p.description,
dup: d.dup ? d.where : false,
});
}
← 3085a5ef auto-save: 2026-06-25T08:02:44 (3 files) — shopify/scripts/c
·
back to Designer Wallcoverings
·
Scrub residual Brewster/York from 44 product descriptions (b 64278b92 →