← back to Designer Wallcoverings
feat(schumacher): feed-first collection extraction returns real products
57b79b4206fd4d626bb3c9a597b3a06bb17bf1f2 · 2026-06-09 15:21:27 -0700 · Steve
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
M DW-Programming/ImportNewSkufromURL/lib/scrapers/schumacher-collections-scraper.ts
Diff
commit 57b79b4206fd4d626bb3c9a597b3a06bb17bf1f2
Author: Steve <steve@designerwallcoverings.com>
Date: Tue Jun 9 15:21:27 2026 -0700
feat(schumacher): feed-first collection extraction returns real products
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
.../lib/scrapers/schumacher-collections-scraper.ts | 338 +++++++++++----------
1 file changed, 170 insertions(+), 168 deletions(-)
diff --git a/DW-Programming/ImportNewSkufromURL/lib/scrapers/schumacher-collections-scraper.ts b/DW-Programming/ImportNewSkufromURL/lib/scrapers/schumacher-collections-scraper.ts
index eb453c64..50a4040b 100644
--- a/DW-Programming/ImportNewSkufromURL/lib/scrapers/schumacher-collections-scraper.ts
+++ b/DW-Programming/ImportNewSkufromURL/lib/scrapers/schumacher-collections-scraper.ts
@@ -1,201 +1,203 @@
/**
- * Schumacher Collections Scraper
- * Auto-generated from database configuration
+ * Schumacher Collections Scraper — FEED-FIRST (Next.js SSR JSON)
+ *
+ * schumacher.com is a Next.js SPA. The catalog grid is server-rendered into the
+ * `__NEXT_DATA__` blob as `props.pageProps.ssrProductFlatResults.content[]` —
+ * a structured JSON array of REAL products (no browser / Algolia needed). We
+ * fetch the category page, parse that JSON, and group products by their source
+ * `collection` so the loader sees an ARRAY of collection objects each carrying a
+ * `.products[]` array (the kravet contract).
+ *
+ * Working catalog slugs: /catalog/wallcoverings (4649), /catalog/fabrics, …
+ * The given URL's filters/slug are honored; if the URL yields 0, we retry the
+ * canonical /catalog/wallcoverings feed.
+ *
+ * NO fake/fallback data (CLAUDE.md): if the feed returns nothing, return [].
*/
-import puppeteer from 'puppeteer';
-import * as dotenv from 'dotenv';
-import { brightDataProxy } from '../brightdata-proxy';
-import { loadVendorSchema, extractWithSchema, getVendorWaitTime } from '../schema-helper';
-import { getVendorTimeout } from '../vendor-timeouts';
-import { ensureArray, safeLength } from '../scraper-wrapper';
-// Load environment variables
-dotenv.config({ path: '.env.local' });
+const SCH_UA =
+ 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
-export interface SchumacherProduct {
- title: string;
+export interface SchumacherCollectionProduct {
name: string;
+ title: string;
pattern: string;
+ color: string;
sku: string;
code: string;
+ productId: string;
url: string;
+ href: string;
+ image?: string;
+ imageUrl?: string;
vendor: string;
vendor_id: string;
category: string;
collection: string;
source: string;
extracted_at: string;
+}
+
+export interface SchumacherCollection {
+ name: string;
+ title: string;
+ url: string;
+ href: string;
+ productCount: number;
image?: string;
- color?: string;
- specifications?: Record<string, string>;
- pricing?: {
- price: string;
- unit: string;
- };
+ products: SchumacherCollectionProduct[];
+ vendor: string;
+ vendor_id: string;
+ extracted_at: string;
}
-export async function scrapeSchumacherCollections(url?: string, maxProducts: number = 100): Promise<SchumacherProduct[]> {
-
- // Load vendor schema from database
- const schema = await loadVendorSchema('schumacher');
- const waitTime = await getVendorWaitTime('schumacher');
- // Get URL from database if not provided
- const targetUrl = url || await getUrlFromDatabase('schumacher', 'collections');
-
- console.log('🎨 Scraping Schumacher collections from:', targetUrl);
-
- // Get BrightData proxy settings
- const proxySettings = brightDataProxy.getProxySettings();
- if (!proxySettings && false) {
- // Proxy is now optional - using direct connection for all scraping');
- }
-
- // proxySettings already declared
-
- const launchArgs = [
- '--disable-gpu',
- '--no-sandbox',
- '--disable-dev-shm-usage',
- '--disable-blink-features=AutomationControlled',
- '--ignore-certificate-errors'
- ];
-
- if (proxySettings) {
- launchArgs.push(`--proxy-server=${proxySettings.server}`);
+/** Pull the JSON object inside <script id="__NEXT_DATA__">…</script>. */
+function extractNextData(html: string): any | null {
+ const m = html.match(
+ /<script id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/
+ );
+ if (!m) return null;
+ try {
+ return JSON.parse(m[1]);
+ } catch {
+ return null;
}
+}
- const browser = await puppeteer.launch({
- headless: true,
- args: launchArgs,
- ignoreHTTPSErrors: true
+/** Map a category page URL to its SSR product content array. */
+async function fetchFlatContent(url: string): Promise<any[]> {
+ const res = await fetch(url, {
+ headers: { 'User-Agent': SCH_UA, Accept: 'text/html' },
});
-
- try {
- const page = await browser.newPage();
-
- if (proxySettings) {
- await page.authenticate({
- username: proxySettings.username,
- password: proxySettings.password
- });
- }
-
- await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
- await page.setViewport({ width: 1920, height: 1080 });
-
- // Use vendor-specific timeout configuration
- const timeout = getVendorTimeout('schumacher', 'collections');
-
-
- page.setDefaultNavigationTimeout(timeout);
- page.setDefaultTimeout(timeout);
- console.log('📄 Loading page...');
- await page.goto(targetUrl, {
- waitUntil: 'domcontentloaded',
- timeout: timeout
- });
-
- await page.waitForTimeout(waitTime || 5000);
-
- // Extract products
- const products = await page.evaluate((limit) => {
- const results: any[] = [];
-
- // Common selectors for products
- const selectors = [
- '.product-item',
- '.product-card',
- '.product',
- 'article.product',
- '.grid-item'
- ];
-
- let productElements: Element[] = [];
- for (const selector of selectors) {
- const elements = Array.from(document.querySelectorAll(selector));
- if (elements.length > 0) {
- productElements = elements;
- break;
- }
- }
-
- // Extract data from each product
- for (let i = 0; i < Math.min(limit, productElements.length); i++) {
- const element = productElements[i];
-
- const title = element.querySelector('.product-title, .product-name, h3, h4')?.textContent?.trim() || '';
- const link = element.querySelector('a')?.getAttribute('href') || '';
- const image = element.querySelector('img')?.getAttribute('src') || '';
- const sku = element.querySelector('.sku, .product-sku')?.textContent?.trim() || `SKU-${i + 1}`;
-
- if (title) {
- results.push({
- title,
- name: title,
- pattern: title.split(' - ')[0] || title,
- sku,
- code: sku,
- url: link.startsWith('http') ? link : `https://schumacher.com/catalog/1?_rv=false&_sg=1&gridSize=md&sort=arrivalDate%2Cdesc&sort=itemNumber%2Cdesc${link}`,
- vendor: 'Schumacher',
- vendor_id: 'schumacher',
- category: 'wallcovering',
- collection: 'Schumacher Collection',
- source: 'product-grid',
- image: image.startsWith('http') ? image : image.startsWith('//') ? `https:${image}` : undefined,
- extracted_at: new Date().toISOString()
- });
- }
- }
-
- return results;
- }, maxProducts);
-
- const safeProducts = Array.isArray(products) ? products : [];
- console.log(`📦 Found ${safeProducts?.length || 0} products`);
- return safeProducts;
-
- } catch (error) {
- console.error('❌ Error scraping Schumacher:', error.message);
+ if (!res.ok) {
+ console.error(`❌ Schumacher page fetch failed: HTTP ${res.status} for ${url}`);
return [];
- } finally {
- await browser.close();
}
+ const html = await res.text();
+ const data = extractNextData(html);
+ const flat = data?.props?.pageProps?.ssrProductFlatResults;
+ const content: any[] = Array.isArray(flat?.content) ? flat.content : [];
+ console.log(
+ `📄 Schumacher SSR: ${url} → total ${flat?.totalElements ?? 0}, content ${content.length}`
+ );
+ return content;
}
-// Helper function to get URL from PostgreSQL database
-async function getUrlFromDatabase(vendorId: string, scraperType: string): Promise<string> {
- try {
- const { queryOne } = await import('../database/postgres-client');
-
- const field = scraperType === 'new-products' ? 'new_products_url' :
- scraperType === 'general' ? 'catalog_url' :
- 'collections_url';
-
- const query = `SELECT ${field} FROM vendors WHERE vendor_id = $1`;
- const result = await queryOne(query, [vendorId]);
-
- if (!result || !result[field]) {
- throw new Error(`No URL found for ${vendorId} ${scraperType}`);
+export async function scrapeSchumacherCollections(params?: {
+ url?: string;
+ maxResults?: number;
+}): Promise<SchumacherCollection[]> {
+ const maxProducts = params?.maxResults || 20;
+ const targetUrl = params?.url || 'https://schumacher.com/catalog/wallcoverings';
+
+ console.log('🎨 Schumacher feed-first collections scrape:', targetUrl);
+
+ let content = await fetchFlatContent(targetUrl);
+
+ // Retry canonical wallcoverings catalog if the given URL is empty / wrong slug.
+ if (content.length === 0 && !/\/catalog\/wallcoverings/.test(targetUrl)) {
+ console.log('↩️ Empty feed; retrying canonical /catalog/wallcoverings');
+ content = await fetchFlatContent('https://schumacher.com/catalog/wallcoverings');
+ }
+
+ if (content.length === 0) return [];
+
+ const now = new Date().toISOString();
+ const seen = new Set<string>();
+ const flatProducts: SchumacherCollectionProduct[] = [];
+
+ for (const hit of content) {
+ if (flatProducts.length >= maxProducts) break;
+ const sku: string = String(hit.itemNumber || hit.id || '').trim();
+ const id = String(hit.id || hit.itemNumber || '');
+ const dedupeKey = sku || id;
+ if (!dedupeKey || seen.has(dedupeKey)) continue;
+ seen.add(dedupeKey);
+
+ const pattern = String(hit.name || '').trim();
+ const color = String(hit.colorName || '').trim();
+ const img = hit.imageUrl || hit.images?.[0]?.largeUrl || hit.images?.[0]?.thumbnailUrl;
+ const collName = (hit.collection?.name || hit.category?.name || 'Schumacher')
+ .toString()
+ .trim();
+ const collSlug = hit.collection?.slug || '';
+ const href = `https://schumacher.com/catalog/products/${id}`;
+
+ if (!pattern) continue;
+
+ flatProducts.push({
+ name: pattern,
+ title: pattern,
+ pattern,
+ color,
+ sku,
+ code: sku,
+ productId: id,
+ url: href,
+ href,
+ image: img,
+ imageUrl: img,
+ vendor: 'Schumacher',
+ vendor_id: 'schumacher',
+ category: hit.category?.name || 'Wallcovering',
+ collection: collName,
+ source: 'next-ssr-feed',
+ extracted_at: now,
+ // group hint
+ ...({ _collSlug: collSlug } as any),
+ } as any);
+ }
+
+ console.log(`✅ Schumacher feed returned ${flatProducts.length} real products`);
+ if (flatProducts.length === 0) return [];
+
+ // Group into collections by `collection.name`; each carries a re-queryable URL.
+ const collectionMap = new Map<string, SchumacherCollection>();
+ for (const p of flatProducts) {
+ const collName = p.collection || 'Schumacher';
+ const collSlug = (p as any)._collSlug;
+ delete (p as any)._collSlug;
+
+ if (!collectionMap.has(collName)) {
+ // Schumacher exposes per-collection catalog pages at /catalog/<slug>.
+ const collUrl = collSlug
+ ? `https://schumacher.com/catalog/${collSlug}`
+ : targetUrl;
+ collectionMap.set(collName, {
+ name: collName,
+ title: collName,
+ url: collUrl,
+ href: collUrl,
+ productCount: 0,
+ image: p.image,
+ products: [],
+ vendor: 'Schumacher',
+ vendor_id: 'schumacher',
+ extracted_at: now,
+ });
}
-
- return result[field];
- } catch (error) {
- console.error('Error querying PostgreSQL:', error);
- throw new Error(`Database error: ${error.message}`);
+ const coll = collectionMap.get(collName)!;
+ coll.products.push(p);
+ coll.productCount = coll.products.length;
+ if (!coll.image && p.image) coll.image = p.image;
}
+
+ const collections = Array.from(collectionMap.values());
+ console.log(`📦 Grouped into ${collections.length} collection(s)`);
+ return collections;
}
// Export for direct testing
if (require.main === module) {
(async () => {
- try {
- const products = await scrapeSchumacherCollections('https://schumacher.com/catalog/1?_rv=false&_sg=1&gridSize=md&sort=arrivalDate%2Cdesc&sort=itemNumber%2Cdesc', 10);
- console.log('\n=== SCHUMACHER COLLECTIONS PRODUCTS ===\n');
- products.forEach((product, index) => {
- console.log(`${index + 1}. ${product.title} (SKU: ${product.sku})`);
- });
- } catch (error) {
- console.error('Failed:', error);
- }
+ const collections = await scrapeSchumacherCollections({
+ url: 'https://schumacher.com/catalog/wallcoverings',
+ maxResults: 10,
+ });
+ console.log('\n=== SCHUMACHER COLLECTIONS ===\n');
+ collections.forEach((c, i) => {
+ console.log(`${i + 1}. ${c.name} (${c.products.length})`);
+ c.products.slice(0, 2).forEach(p => console.log(` - ${p.name} | ${p.color} | ${p.sku}`));
+ });
})();
-}
\ No newline at end of file
+}
← 9b0f6949 rotation pre-stage: DW-Agents DATABASE_URL env-first so rota
·
back to Designer Wallcoverings
·
feat(designers-guild): feed-first collection extraction retu 6e401a7d →