← back to Designer Wallcoverings
feat(dedar): feed-first collection extraction returns real products
06ea863b7eeb4c282c41c4d45155a996431e924d · 2026-06-09 15:21:35 -0700 · Steve
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
M DW-Programming/ImportNewSkufromURL/lib/scrapers/dedar-collections-scraper.ts
Diff
commit 06ea863b7eeb4c282c41c4d45155a996431e924d
Author: Steve <steve@designerwallcoverings.com>
Date: Tue Jun 9 15:21:35 2026 -0700
feat(dedar): feed-first collection extraction returns real products
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
.../lib/scrapers/dedar-collections-scraper.ts | 273 +++++++++++++--------
1 file changed, 172 insertions(+), 101 deletions(-)
diff --git a/DW-Programming/ImportNewSkufromURL/lib/scrapers/dedar-collections-scraper.ts b/DW-Programming/ImportNewSkufromURL/lib/scrapers/dedar-collections-scraper.ts
index c8637df9..e84bff2a 100644
--- a/DW-Programming/ImportNewSkufromURL/lib/scrapers/dedar-collections-scraper.ts
+++ b/DW-Programming/ImportNewSkufromURL/lib/scrapers/dedar-collections-scraper.ts
@@ -1,120 +1,191 @@
-import { brightDataProxy } from '../brightdata-proxy';
/**
- * Dedar Collections Scraper Skill
- * Focuses specifically on collections/categories scraping for Dedar
+ * Dedar Collections Scraper — FEED-FIRST (BigCommerce Stencil SSR HTML)
+ *
+ * dedar.com runs BigCommerce (Stencil). The product grid is server-rendered as
+ * `<ul class="productGrid">` containing `<li class="product" data-sku="…">`
+ * cards (initially display:none, hydrated client-side — but the data is in the
+ * HTML). No browser needed; we fetch + regex the cards:
+ * - sku → `data-sku` on the <li>
+ * - name → card `aria-label` ("Amoir Libre Wall, €0,00") → text before comma
+ * - color → `data-alt` ("032|Green") → segment after the pipe
+ * - href → card-figure__link href (e.g. /amoir-libre-wall/)
+ * - image → `data-src` (BigCommerce CDN, {:size} templated → fill 500x500)
+ *
+ * Returns an ARRAY of collection objects each carrying `.products[]` (kravet
+ * contract). The category page is treated as one collection.
+ *
+ * NO fake/fallback data (CLAUDE.md): if the page yields nothing, return [].
*/
-import { scrapeDedarCollections } from '../lib/scrapers/dedar-collections-scraper';
-import { ensureArray, safeLength } from '../scraper-wrapper';
+const DEDAR_UA =
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
-export interface DedarCollectionsResult {
+export interface DedarCollectionProduct {
+ name: string;
+ title: string;
+ pattern: string;
+ color: string;
+ sku: string;
+ code: string;
+ url: string;
+ href: string;
+ image?: string;
+ imageUrl?: string;
vendor: string;
- timestamp: string;
- collections: any[];
- totals: {
- collections: number;
- };
+ vendor_id: string;
+ category: string;
+ collection: string;
+ source: string;
+ extracted_at: string;
}
-/**
- * Main collections skill function for Dedar
- */
-export async function runDedarCollectionsSkill(options: {
- maxCollections?: number;
-} = {}): Promise<DedarCollectionsResult> {
- const { maxCollections = 15 } = options;
-
- console.log('🎯 DEDAR COLLECTIONS SKILL ACTIVATED');
- console.log('━'.repeat(60));
- console.log(`📂 Max collections: ${maxCollections}`);
- console.log('');
+export interface DedarCollection {
+ name: string;
+ title: string;
+ url: string;
+ href: string;
+ productCount: number;
+ image?: string;
+ products: DedarCollectionProduct[];
+ vendor: string;
+ vendor_id: string;
+ extracted_at: string;
+}
- const result: DedarCollectionsResult = {
- vendor: 'Dedar',
- timestamp: new Date().toISOString(),
- collections: [],
- totals: {
- collections: 0
- }
- };
+function decodeEntities(s: string): string {
+ return (s || '')
+ .replace(/&/g, '&')
+ .replace(/=|=/g, '=')
+ .replace(/'|'/g, "'")
+ .replace(/"/g, '"')
+ .replace(/</g, '<')
+ .replace(/>/g, '>')
+ .replace(/\s+/g, ' ')
+ .trim();
+}
+/** Derive a friendly collection title from the category URL path. */
+function collectionNameFromUrl(url: string): string {
try {
- console.log('📂 Scraping collections from Dedar...');
-
- const collections = await scrapeDedarCollections();
-
- // Apply limit
- const limitedCollections = collections.slice(0, maxCollections);
- result.collections = limitedCollections;
- result.totals.collections = limitedCollections.length;
-
- console.log(`✅ COLLECTIONS: Found ${limitedCollections.length} collections`);
-
- if (limitedCollections.length > 0) {
- console.log('\n📂 Sample collections:');
- limitedCollections.slice(0, 3).forEach((c, i) => {
- const name = c.name || c.title || c.collectionName || `Collection ${i + 1}`;
- console.log(`${i + 1}. ${name}`);
- });
- }
+ const seg = new URL(url).pathname.split('/').filter(Boolean).pop() || '';
+ if (!seg) return 'Dedar Wallcovering';
+ return seg
+ .split('-')
+ .map(w => w.charAt(0).toUpperCase() + w.slice(1))
+ .join(' ');
+ } catch {
+ return 'Dedar Wallcovering';
+ }
+}
- console.log('\n📊 DEDAR COLLECTIONS SUMMARY');
- console.log('━'.repeat(60));
- console.log(`✅ Collections found: ${result.totals.collections}`);
+export async function scrapeDedarCollections(params?: {
+ url?: string;
+ maxResults?: number;
+}): Promise<DedarCollection[]> {
+ const maxProducts = params?.maxResults || 20;
+ const targetUrl = params?.url || 'https://www.dedar.com/products/wallcovering';
- return result;
+ console.log('🎨 Dedar feed-first (BigCommerce SSR) collections scrape:', targetUrl);
- } catch (error) {
- console.error('❌ Dedar Collections skill error:', error);
- console.error('Error details:', error.message);
- return result;
+ const res = await fetch(targetUrl, {
+ headers: { 'User-Agent': DEDAR_UA, Accept: 'text/html' },
+ });
+ if (!res.ok) {
+ console.error(`❌ Dedar page fetch failed: HTTP ${res.status}`);
+ return [];
}
-}
+ const html = await res.text();
-/**
- * Collections skill registration object
- */
-export const dedarCollectionsSkill = {
- name: 'Dedar Collections Scraper',
- vendor: 'dedar',
- description: 'Collections and categories scraper for Dedar',
-
- // Main execute function
- execute: async (options = {}) => {
- return await runDedarCollectionsSkill(options);
- },
-
- // Direct collections function
- collections: async (maxCollections = 15) => {
- const result = await runDedarCollectionsSkill({ maxCollections });
- return result.collections;
- },
-
- // Test function to verify skill is working
- test: async () => {
- console.log('🧪 Testing Dedar Collections skill...');
- const result = await runDedarCollectionsSkill({ maxCollections: 3 });
-
- const success = result.totals.collections > 0;
- console.log(success ? '✅ Test PASSED' : '❌ Test FAILED');
- return success;
+ const now = new Date().toISOString();
+ const seen = new Set<string>();
+ const products: DedarCollectionProduct[] = [];
+
+ // Each card opens with: <li class="product" data-sku="…" data-id="…">
+ const cardRe = /<li class="product" data-sku="([^"]*)"[^>]*>([\s\S]*?)(?=<li class="product" data-sku=|<\/ul>)/g;
+ let m: RegExpExecArray | null;
+ while ((m = cardRe.exec(html)) !== null) {
+ if (products.length >= maxProducts) break;
+ const sku = decodeEntities(m[1]);
+ const seg = m[2];
+ if (!sku || seen.has(sku)) continue;
+
+ const hrefM = seg.match(/class="card-figure__link"[^>]*href="([^"]+)"|href="([^"]+)"[^>]*class="card-figure__link"/);
+ let href = hrefM ? decodeEntities(hrefM[1] || hrefM[2]) : '';
+ // aria-label sometimes carries the href; also pull from any anchor.
+ if (!href) {
+ const anyHref = seg.match(/<a[^>]*href="(https?:\/\/[^"]+)"/);
+ if (anyHref) href = decodeEntities(anyHref[1]);
+ }
+
+ const labelM = seg.match(/aria-label="([^"]+)"/);
+ let name = labelM ? decodeEntities(labelM[1]).split(',')[0].trim() : '';
+
+ const altM = seg.match(/data-alt="([^"]*)"/);
+ const alt = altM ? decodeEntities(altM[1]) : '';
+ // data-alt is "032|Green" → colour is the part after the pipe.
+ let color = '';
+ if (alt.includes('|')) color = alt.split('|').slice(1).join(' ').trim();
+ else color = alt.trim();
+
+ let image: string | undefined;
+ const srcM = seg.match(/data-src="([^"]+)"/) || seg.match(/<img[^>]*src=([^ >]+)/);
+ if (srcM) {
+ image = decodeEntities(srcM[1]).replace('{:size}', '500x500');
+ if (image.startsWith('//')) image = 'https:' + image;
+ }
+
+ if (!name) name = collectionNameFromUrl(href || targetUrl);
+ if (!name) continue;
+ seen.add(sku);
+
+ products.push({
+ name,
+ title: name,
+ pattern: name,
+ color,
+ sku,
+ code: sku,
+ url: href,
+ href,
+ image,
+ imageUrl: image,
+ vendor: 'Dedar',
+ vendor_id: 'dedar',
+ category: 'Wallcovering',
+ collection: collectionNameFromUrl(targetUrl),
+ source: 'bigcommerce-ssr-html',
+ extracted_at: now,
+ });
}
-};
-// Export default for direct execution
-export default dedarCollectionsSkill;
+ console.log(`✅ Dedar returned ${products.length} real products`);
+ if (products.length === 0) return [];
+
+ const collName = collectionNameFromUrl(targetUrl);
+ const collection: DedarCollection = {
+ name: collName,
+ title: collName,
+ url: targetUrl,
+ href: targetUrl,
+ productCount: products.length,
+ image: products[0].image,
+ products,
+ vendor: 'Dedar',
+ vendor_id: 'dedar',
+ extracted_at: now,
+ };
+ console.log('📦 Grouped into 1 collection(s)');
+ return [collection];
+}
-// Allow direct execution
+// Export for direct testing
if (require.main === module) {
- console.log('🚀 Running Dedar Collections skill directly...\n');
-
- runDedarCollectionsSkill({
- maxCollections: 10
- }).then(result => {
- console.log('\n✨ Dedar Collections skill completed successfully!');
- process.exit(0);
- }).catch(error => {
- console.error('Fatal error:', error);
- process.exit(1);
- });
-}
\ No newline at end of file
+ (async () => {
+ const collections = await scrapeDedarCollections({ maxResults: 10 });
+ console.log('\n=== DEDAR COLLECTIONS ===\n');
+ collections.forEach((c, i) => {
+ console.log(`${i + 1}. ${c.name} (${c.products.length})`);
+ c.products.slice(0, 3).forEach(p => console.log(` - ${p.name} | ${p.color} | ${p.sku}`));
+ });
+ })();
+}
← 160fad08 feat(farrow-ball): feed-first collection extraction returns
·
back to Designer Wallcoverings
·
PJ vendor_catalog spec scraper: tear-sheet specs for all phi 6b33ebc1 →