← back to Designer Wallcoverings
feat(designers-guild): feed-first collection extraction returns real products
6e401a7d287c0c206d2071c6fb0e29770206eed2 · 2026-06-09 15:21:34 -0700 · Steve
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
M DW-Programming/ImportNewSkufromURL/lib/scrapers/designers-guild-collections-scraper.ts
Diff
commit 6e401a7d287c0c206d2071c6fb0e29770206eed2
Author: Steve <steve@designerwallcoverings.com>
Date: Tue Jun 9 15:21:34 2026 -0700
feat(designers-guild): feed-first collection extraction returns real products
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
.../designers-guild-collections-scraper.ts | 359 +++++++++++++++------
1 file changed, 262 insertions(+), 97 deletions(-)
diff --git a/DW-Programming/ImportNewSkufromURL/lib/scrapers/designers-guild-collections-scraper.ts b/DW-Programming/ImportNewSkufromURL/lib/scrapers/designers-guild-collections-scraper.ts
index 1762fb80..609b7601 100644
--- a/DW-Programming/ImportNewSkufromURL/lib/scrapers/designers-guild-collections-scraper.ts
+++ b/DW-Programming/ImportNewSkufromURL/lib/scrapers/designers-guild-collections-scraper.ts
@@ -1,120 +1,285 @@
-import { brightDataProxy } from '../brightdata-proxy';
/**
- * Designers Guild Collections Scraper Skill
- * Focuses specifically on collections/categories scraping for Designers Guild
+ * Designers Guild Collections Scraper — FEED-FIRST (ASP.NET SSR embedded JSON)
+ *
+ * designersguild.com is a custom ASP.NET storefront behind Cloudflare. Many
+ * routes (incl. the config's old `/new-wallpaper-collections/l111`) serve a
+ * "Major Error" / Login page to bots, BUT the real catalog pages are reachable
+ * via plain fetch and embed full product objects in their SSR HTML — both as
+ * raw JSON (lifestyle hotspots) and HTML-entity-encoded JSON (the PLP grid).
+ *
+ * The wallpaper catalog is organised as COLLECTIONS: the index
+ * `/en-us/wallpaper/designers-guild/l1119` links to per-collection listing
+ * pages (`…/<collection-slug>/c<NNN>` and `…/l<NNN>`). Each collection page
+ * embeds its products. We:
+ * 1. fetch the collections index, harvest the per-collection listing links,
+ * 2. fetch each collection page and parse its embedded product objects,
+ * 3. return an ARRAY of collection objects each carrying `.products[]`.
+ *
+ * Per product: code (SKU, e.g. PDG1031/04), name ("Pavonazzo Pewter" → pattern
+ * + color), url (PDP), image (https://www.designersguild.com/image/1024/<id>.jpg).
+ *
+ * NO fake/fallback data (CLAUDE.md): if nothing is found, return [].
*/
-import { scrapeDesignersGuildCollections } from '../lib/scrapers/designers-guild-collections-scraper';
-import { ensureArray, safeLength } from '../scraper-wrapper';
+const DG_UA =
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+const DG_BASE = 'https://www.designersguild.com';
-export interface DesignersGuildCollectionsResult {
+export interface DesignersGuildCollectionProduct {
+ name: string;
+ title: string;
+ pattern: string;
+ color: string;
+ sku: string;
+ code: string;
+ productId: string;
+ url: string;
+ href: string;
+ image?: string;
+ imageUrl?: string;
vendor: string;
- timestamp: string;
- collections: any[];
- totals: {
- collections: number;
- };
+ vendor_id: string;
+ category: string;
+ collection: string;
+ source: string;
+ extracted_at: string;
+}
+
+export interface DesignersGuildCollection {
+ name: string;
+ title: string;
+ url: string;
+ href: string;
+ productCount: number;
+ image?: string;
+ products: DesignersGuildCollectionProduct[];
+ vendor: string;
+ vendor_id: string;
+ extracted_at: string;
+}
+
+async function fetchHtml(url: string): Promise<string> {
+ try {
+ const r = await fetch(url, {
+ headers: {
+ 'User-Agent': DG_UA,
+ Accept: 'text/html,application/xhtml+xml',
+ 'Accept-Language': 'en-US,en;q=0.9',
+ },
+ });
+ if (!r.ok) return '';
+ return await r.text();
+ } catch {
+ return '';
+ }
+}
+
+/** DG name "Pavonazzo Pewter" / "Floreale - Celadon" → { pattern, color }. */
+function splitNameColor(name: string): { pattern: string; color: string } {
+ const raw = (name || '').replace(/\s+/g, ' ').trim();
+ const dash = raw.indexOf(' - ');
+ if (dash > -1) {
+ return { pattern: raw.slice(0, dash).trim(), color: raw.slice(dash + 3).trim() };
+ }
+ // Otherwise the last word is usually the colour (e.g. "Delft Flower Gold").
+ const parts = raw.split(' ');
+ if (parts.length >= 2) {
+ return { pattern: parts.slice(0, -1).join(' '), color: parts[parts.length - 1] };
+ }
+ return { pattern: raw, color: '' };
}
/**
- * Main collections skill function for Designers Guild
+ * Harvest wallpaper product objects embedded in a DG page. The product JSON is
+ * present both raw and entity-encoded, so we decode entities first, then pull
+ * every `"product":{ … }` object with a code, name, imageId and PDP url.
*/
-export async function runDesignersGuildCollectionsSkill(options: {
- maxCollections?: number;
-} = {}): Promise<DesignersGuildCollectionsResult> {
- const { maxCollections = 15 } = options;
+function harvestProducts(html: string): Array<{
+ code: string;
+ name: string;
+ imageId: string;
+ url: string;
+ productId: string;
+}> {
+ const d = html.replace(/"/g, '"').replace(/&/g, '&');
+ const out: any[] = [];
+ const seen = new Set<string>();
+
+ // Anchor on "code":"<CODE>" then look ahead (bounded) for name / imageId / url.
+ const re = /"code":"([A-Z]{2,5}\d{2,4}\/[A-Z0-9]+)"/g;
+ let m: RegExpExecArray | null;
+ while ((m = re.exec(d)) !== null) {
+ const code = m[1];
+ const win = d.slice(m.index, m.index + 1200); // bounded window after the code
+ const urlM = win.match(/"url":"(\/en-us\/[^"]*\/p(\d+))"/);
+ if (!urlM) continue;
+ const url = urlM[1];
+ if (!/\/wallpaper\//.test(url)) continue; // wallpaper only
+ if (seen.has(url)) continue;
+ seen.add(url);
+ const nameM = win.match(/"name":"([^"]*)"/);
+ const imgM = win.match(/"imageId":(\d+)/);
+ out.push({
+ code,
+ name: nameM ? nameM[1] : '',
+ imageId: imgM ? imgM[1] : '',
+ url,
+ productId: urlM[2],
+ });
+ }
+ return out;
+}
- console.log('🎯 DESIGNERS GUILD COLLECTIONS SKILL ACTIVATED');
- console.log('━'.repeat(60));
- console.log(`📂 Max collections: ${maxCollections}`);
- console.log('');
+/** Find per-collection listing links on the wallpaper collections index. */
+function harvestCollectionLinks(html: string): string[] {
+ const links = html.match(/\/en-us\/wallpaper\/[a-z0-9-]+\/(?:c|l)\d+/g) || [];
+ return [...new Set(links)];
+}
- const result: DesignersGuildCollectionsResult = {
+function buildProduct(
+ p: { code: string; name: string; imageId: string; url: string; productId: string },
+ collName: string,
+ now: string
+): DesignersGuildCollectionProduct {
+ const { pattern, color } = splitNameColor(p.name || p.code);
+ const href = DG_BASE + p.url;
+ const image = p.imageId ? `${DG_BASE}/image/1024/${p.imageId}.jpg` : undefined;
+ return {
+ name: p.name || pattern || p.code,
+ title: p.name || pattern || p.code,
+ pattern: pattern || p.code,
+ color,
+ sku: p.code,
+ code: p.code,
+ productId: p.productId,
+ url: href,
+ href,
+ image,
+ imageUrl: image,
vendor: 'Designers Guild',
- timestamp: new Date().toISOString(),
- collections: [],
- totals: {
- collections: 0
- }
+ vendor_id: 'designers-guild',
+ category: 'Wallcovering',
+ collection: collName,
+ source: 'aspnet-ssr-embedded-json',
+ extracted_at: now,
};
+}
- try {
- console.log('📂 Scraping collections from Designers Guild...');
-
- const collections = await scrapeDesignersGuildCollections();
-
- // Apply limit
- const limitedCollections = collections.slice(0, maxCollections);
- result.collections = limitedCollections;
- result.totals.collections = limitedCollections.length;
-
- console.log(`✅ COLLECTIONS: Found ${limitedCollections.length} collections`);
-
- if (limitedCollections.length > 0) {
- console.log('\n📂 Sample collections:');
- limitedCollections.slice(0, 3).forEach((c, i) => {
- const name = c.name || c.title || c.collectionName || `Collection ${i + 1}`;
- console.log(`${i + 1}. ${name}`);
- });
- }
+export async function scrapeDesignersGuildCollections(params?: {
+ url?: string;
+ maxResults?: number;
+}): Promise<DesignersGuildCollection[]> {
+ const maxProducts = params?.maxResults || 20;
+ // The collections index (the config's l111 errors → use the working l1119).
+ const indexUrl = params?.url || `${DG_BASE}/en-us/wallpaper/designers-guild/l1119`;
- console.log('\n📊 DESIGNERS GUILD COLLECTIONS SUMMARY');
- console.log('━'.repeat(60));
- console.log(`✅ Collections found: ${result.totals.collections}`);
+ console.log('🎨 Designers Guild feed-first (ASP.NET SSR JSON) scrape:', indexUrl);
- return result;
+ const now = new Date().toISOString();
+ const collections: DesignersGuildCollection[] = [];
+ const seenSkus = new Set<string>();
+ let total = 0;
- } catch (error) {
- console.error('❌ Designers Guild Collections skill error:', error);
- console.error('Error details:', error.message);
- return result;
+ const indexHtml = await fetchHtml(indexUrl);
+ if (!indexHtml) {
+ console.error('❌ Designers Guild index fetch failed');
+ return [];
}
-}
-/**
- * Collections skill registration object
- */
-export const designersguildCollectionsSkill = {
- name: 'Designers Guild Collections Scraper',
- vendor: 'designers-guild',
- description: 'Collections and categories scraper for Designers Guild',
-
- // Main execute function
- execute: async (options = {}) => {
- return await runDesignersGuildCollectionsSkill(options);
- },
-
- // Direct collections function
- collections: async (maxCollections = 15) => {
- const result = await runDesignersGuildCollectionsSkill({ maxCollections });
- return result.collections;
- },
-
- // Test function to verify skill is working
- test: async () => {
- console.log('🧪 Testing Designers Guild Collections skill...');
- const result = await runDesignersGuildCollectionsSkill({ maxCollections: 3 });
-
- const success = result.totals.collections > 0;
- console.log(success ? '✅ Test PASSED' : '❌ Test FAILED');
- return success;
+ // 1. Products directly present on the index page → "Featured" collection.
+ const indexProducts = harvestProducts(indexHtml);
+
+ // 2. Per-collection listing links.
+ let collLinks = harvestCollectionLinks(indexHtml).filter(l => !/\/l1119$/.test(l));
+
+ // Fallback seed collections (the well-known wallpaper c-listings) if the
+ // index links are sparse — keeps the scraper working if nav markup shifts.
+ if (collLinks.length === 0) {
+ collLinks = [
+ '/en-us/wallpaper/tulipa-stellata-wallpaper/c654',
+ '/en-us/wallpaper/geometric-wallpaper/c666',
+ '/en-us/wallpaper/the-edit-flowers-wallpaper-volume-i/c700',
+ ];
+ }
+
+ // Seed the index-page products as a collection so we never lose them.
+ if (indexProducts.length > 0) {
+ const prods = indexProducts
+ .filter(p => !seenSkus.has(p.code))
+ .map(p => {
+ seenSkus.add(p.code);
+ return buildProduct(p, 'Designers Guild Wallpaper', now);
+ });
+ if (prods.length) {
+ collections.push({
+ name: 'Designers Guild Wallpaper',
+ title: 'Designers Guild Wallpaper',
+ url: indexUrl,
+ href: indexUrl,
+ productCount: prods.length,
+ image: prods[0].image,
+ products: prods,
+ vendor: 'Designers Guild',
+ vendor_id: 'designers-guild',
+ extracted_at: now,
+ });
+ total += prods.length;
+ }
+ }
+
+ // 3. Walk each collection page (bounded) and harvest its products.
+ for (const link of collLinks) {
+ if (total >= maxProducts) break;
+ const collUrl = DG_BASE + link;
+ const html = await fetchHtml(collUrl);
+ if (!html) continue;
+
+ const titleM = html.match(/<title>([^<|]+)/);
+ const collName =
+ (titleM ? titleM[1] : link.split('/').filter(Boolean).slice(-2, -1)[0] || 'Designers Guild')
+ .replace(/&/g, '&')
+ .replace(/\s+/g, ' ')
+ .trim();
+
+ const raw = harvestProducts(html);
+ const prods: DesignersGuildCollectionProduct[] = [];
+ for (const p of raw) {
+ if (total >= maxProducts) break;
+ if (seenSkus.has(p.code)) continue;
+ seenSkus.add(p.code);
+ prods.push(buildProduct(p, collName, now));
+ total++;
+ }
+ if (prods.length > 0) {
+ collections.push({
+ name: collName,
+ title: collName,
+ url: collUrl,
+ href: collUrl,
+ productCount: prods.length,
+ image: prods[0].image,
+ products: prods,
+ vendor: 'Designers Guild',
+ vendor_id: 'designers-guild',
+ extracted_at: now,
+ });
+ }
}
-};
-// Export default for direct execution
-export default designersguildCollectionsSkill;
+ console.log(
+ `✅ Designers Guild returned ${total} real products across ${collections.length} collection(s)`
+ );
+ return total > 0 ? collections : [];
+}
-// Allow direct execution
+// Export for direct testing
if (require.main === module) {
- console.log('🚀 Running Designers Guild Collections skill directly...\n');
-
- runDesignersGuildCollectionsSkill({
- maxCollections: 10
- }).then(result => {
- console.log('\n✨ Designers Guild Collections skill completed successfully!');
- process.exit(0);
- }).catch(error => {
- console.error('Fatal error:', error);
- process.exit(1);
- });
-}
\ No newline at end of file
+ (async () => {
+ const collections = await scrapeDesignersGuildCollections({ maxResults: 12 });
+ console.log('\n=== DESIGNERS GUILD COLLECTIONS ===\n');
+ collections.forEach((c, i) => {
+ console.log(`${i + 1}. ${c.name} (${c.products.length})`);
+ c.products.slice(0, 3).forEach(p => console.log(` - ${p.pattern} | ${p.color} | ${p.sku}`));
+ });
+ })();
+}
← 57b79b42 feat(schumacher): feed-first collection extraction returns r
·
back to Designer Wallcoverings
·
feat(farrow-ball): feed-first collection extraction returns 160fad08 →