← back to Designer Wallcoverings
auto-save: 2026-07-13T01:52:00 (6 files) — DW-Programming/ImportNewSkufromURL/lib/scrapers/catchii-new-products-scraper.ts DW-Programming/ImportNewSkufromURL/scripts/drain-import-queues.sh pending-approval/boost-filter-consolidation-2026-06-25 scripts/pr-contact-for-price shopify/scripts/cadence/data/upload-restore-2026-07-13.jsonl
4b9ff77334f801ac0b116a966648f950de4bb555 · 2026-07-13 01:52:13 -0700 · Steve Abrams
Files touched
M DW-Programming/ImportNewSkufromURL/lib/scrapers/catchii-new-products-scraper.tsM DW-Programming/ImportNewSkufromURL/scripts/drain-import-queues.shM shopify/scripts/cadence/data/upload-restore-2026-07-13.jsonl
Diff
commit 4b9ff77334f801ac0b116a966648f950de4bb555
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Jul 13 01:52:13 2026 -0700
auto-save: 2026-07-13T01:52:00 (6 files) — DW-Programming/ImportNewSkufromURL/lib/scrapers/catchii-new-products-scraper.ts DW-Programming/ImportNewSkufromURL/scripts/drain-import-queues.sh pending-approval/boost-filter-consolidation-2026-06-25 scripts/pr-contact-for-price shopify/scripts/cadence/data/upload-restore-2026-07-13.jsonl
---
.../lib/scrapers/catchii-new-products-scraper.ts | 82 ++++++++++++----------
.../scripts/drain-import-queues.sh | 4 +-
.../cadence/data/upload-restore-2026-07-13.jsonl | 3 +
3 files changed, 52 insertions(+), 37 deletions(-)
diff --git a/DW-Programming/ImportNewSkufromURL/lib/scrapers/catchii-new-products-scraper.ts b/DW-Programming/ImportNewSkufromURL/lib/scrapers/catchii-new-products-scraper.ts
index f0e3651a..9a8cef5f 100644
--- a/DW-Programming/ImportNewSkufromURL/lib/scrapers/catchii-new-products-scraper.ts
+++ b/DW-Programming/ImportNewSkufromURL/lib/scrapers/catchii-new-products-scraper.ts
@@ -1,66 +1,76 @@
/**
- * Catchii New Products Scraper — FEED-FIRST (WooCommerce Store API, no browser, no proxy)
+ * Catchii New Products Scraper — FEED-FIRST (Shopify products.json, no browser, no proxy)
*
- * Catchii (catchii.com, NL) runs WordPress + WooCommerce with the public Store API:
- * https://www.catchii.com/wp-json/wc/store/v1/products?per_page=100&page=N&category=417
- * Category 417 = "behang" (wallpaper). SKUs look like W100351 / W200111 / B008.
- * Created 2026-06-09 (scraper-creation batch 1).
+ * 2026-07-13 FIX: catchii.com MIGRATED WooCommerce → Shopify. The old Woo Store API
+ * (/wp-json/wc/store/v1/products) now 404s; the site is Shopify (sitemap_products_1.xml
+ * + working /products.json). Re-pointed to the Shopify feed. SKUs look like W100351 / B008.
+ * Woo version kept alongside as .backup-*. Created 2026-06-09; re-platformed here.
*/
export type ScrapeParams = { url?: string; maxResults?: number };
const BASE = 'https://www.catchii.com';
-const WALLPAPER_CATEGORY = 417; // "behang" — wallpaper parent category
const UA =
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
-async function fetchJson(url: string): Promise<any> {
- const ctrl = new AbortController();
- const t = setTimeout(() => ctrl.abort(), 20000);
- try {
- const res = await fetch(url, {
- headers: { 'User-Agent': UA, Accept: 'application/json' },
- redirect: 'follow',
- signal: ctrl.signal,
- });
- if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
- return await res.json();
- } finally {
- clearTimeout(t);
+// Transient 503 retry (Shopify/CF blips) so an unattended monthly run survives a hiccup.
+async function fetchJson(url: string, attempts = 4): Promise<any> {
+ let last = 0;
+ for (let i = 0; i < attempts; i++) {
+ const ctrl = new AbortController();
+ const t = setTimeout(() => ctrl.abort(), 20000);
+ try {
+ const res = await fetch(url, {
+ headers: { 'User-Agent': UA, Accept: 'application/json' },
+ redirect: 'follow',
+ signal: ctrl.signal,
+ });
+ if (res.ok) return await res.json();
+ last = res.status;
+ } catch {
+ last = last || -1;
+ } finally {
+ clearTimeout(t);
+ }
+ if (i < attempts - 1) await new Promise((r) => setTimeout(r, 1500 * (i + 1)));
}
+ throw new Error(`HTTP ${last} for ${url}`);
}
function mapProduct(p: any) {
+ const variant = (p.variants && p.variants[0]) || {};
+ const img = (p.images && p.images[0] && p.images[0].src) || '';
+ const color =
+ (p.options || [])
+ .filter((o: any) => /colou?r|kleur/i.test(o.name))
+ .flatMap((o: any) => o.values)[0] ||
+ (variant.title && variant.title !== 'Default Title' ? variant.title : '');
return {
- name: (p.name || '').trim(),
- color: '',
- sku: (p.sku || '').trim() || p.slug || '',
- href: p.permalink || `${BASE}/product/${p.slug}/`,
- url: p.permalink || `${BASE}/product/${p.slug}/`,
- image: p.images?.[0]?.src || '',
- price:
- p.prices && p.prices.price
- ? String(Number(p.prices.price) / Math.pow(10, p.prices.currency_minor_unit ?? 2))
- : '',
- source: 'catchii',
+ name: (p.title || '').trim(),
+ color: color || '',
+ sku: (variant.sku || '').trim() || String(p.id),
+ href: `${BASE}/products/${p.handle}`,
+ url: `${BASE}/products/${p.handle}`,
+ image: img,
+ price: variant.price ? String(variant.price) : '',
+ source: 'catchii-shopify',
};
}
export async function scrapeCatchiiNewProducts(params?: ScrapeParams) {
const { maxResults = 20 } = params || {};
- console.log('🦜 Scraping Catchii (Woo Store API, wallpaper category)…');
+ console.log('🦜 Scraping Catchii (Shopify products.json)…');
const out: any[] = [];
for (let page = 1; page <= 10 && out.length < maxResults; page++) {
- const data = await fetchJson(
- `${BASE}/wp-json/wc/store/v1/products?per_page=100&page=${page}&category=${WALLPAPER_CATEGORY}&orderby=date&order=desc`
- );
- const items: any[] = Array.isArray(data) ? data : [];
+ const data = await fetchJson(`${BASE}/products.json?limit=250&page=${page}`);
+ const items: any[] = (data && data.products) || [];
+ if (!items.length) break;
for (const p of items) {
out.push(mapProduct(p));
if (out.length >= maxResults) break;
}
- if (items.length < 100) break;
+ if (items.length < 250) break;
}
console.log(`✅ Catchii: ${out.length} products`);
return out;
diff --git a/DW-Programming/ImportNewSkufromURL/scripts/drain-import-queues.sh b/DW-Programming/ImportNewSkufromURL/scripts/drain-import-queues.sh
index c63f2958..96c2d3ad 100755
--- a/DW-Programming/ImportNewSkufromURL/scripts/drain-import-queues.sh
+++ b/DW-Programming/ImportNewSkufromURL/scripts/drain-import-queues.sh
@@ -27,7 +27,9 @@ BUDGET=${DRAIN_BUDGET:-400}
for q in data/scraper-audit/reconcile/*-import-queue.jsonl; do
[ -f "$q" ] || continue
[ "$BUDGET" -le 0 ] && { echo "[drain] daily budget spent — stopping"; break; }
- pending=$(grep -c '"status":"pending"' "$q" 2>/dev/null || echo 0)
+ # grep -c prints "0" AND exits 1 on no-match, so `|| echo 0` produced "0\n0" —
+ # the [ -eq ] test then errored and the skip branch never fired (runner ran on empty queues)
+ pending=$(grep -c '"status":"pending"' "$q" 2>/dev/null); pending=${pending:-0}
if [ "$pending" -eq 0 ]; then echo "[drain] $q: 0 pending — skip"; continue; fi
echo "[drain] $q: $pending pending (budget left: $BUDGET)"
out=$(/opt/homebrew/bin/node scripts/import-queue-runner.js "$q" "$BUDGET" 2>&1)
diff --git a/shopify/scripts/cadence/data/upload-restore-2026-07-13.jsonl b/shopify/scripts/cadence/data/upload-restore-2026-07-13.jsonl
index 34635af4..690f0ee2 100644
--- a/shopify/scripts/cadence/data/upload-restore-2026-07-13.jsonl
+++ b/shopify/scripts/cadence/data/upload-restore-2026-07-13.jsonl
@@ -16,3 +16,6 @@
{"table":"thibaut_catalog","mfr_sku":"T3667","dw_sku":"DWTT-74105","shopify_product_id":"7883398479923","action":"created","activated":true,"published":true,"sampleOnly":false,"status":"ACTIVE","batch_id":"upload-am-2026-07-13T07-40-05-278Z","ts":"2026-07-13T07:40:10.620Z"}
{"table":"thibaut_catalog","mfr_sku":"T3664","dw_sku":"DWTT-74106","shopify_product_id":"7883398512691","action":"created","activated":true,"published":true,"sampleOnly":false,"status":"ACTIVE","batch_id":"upload-am-2026-07-13T07-40-05-278Z","ts":"2026-07-13T07:40:14.353Z"}
{"table":"thibaut_catalog","mfr_sku":"T403","dw_sku":"DWTT-74107","shopify_product_id":"7883398545459","action":"created","activated":true,"published":true,"sampleOnly":false,"status":"ACTIVE","batch_id":"upload-am-2026-07-13T07-40-05-278Z","ts":"2026-07-13T07:40:17.940Z"}
+{"table":"thibaut_catalog","mfr_sku":"T406","dw_sku":"DWTT-74108","shopify_product_id":"7883457167411","action":"created","activated":true,"published":true,"sampleOnly":false,"status":"ACTIVE","batch_id":"upload-am-2026-07-13T08-40-05-671Z","ts":"2026-07-13T08:40:11.771Z"}
+{"table":"thibaut_catalog","mfr_sku":"T409","dw_sku":"DWTT-74109","shopify_product_id":"7883457232947","action":"created","activated":true,"published":true,"sampleOnly":false,"status":"ACTIVE","batch_id":"upload-am-2026-07-13T08-40-05-671Z","ts":"2026-07-13T08:40:15.545Z"}
+{"table":"thibaut_catalog","mfr_sku":"T407","dw_sku":"DWTT-74110","shopify_product_id":"7883457265715","action":"created","activated":true,"published":true,"sampleOnly":false,"status":"ACTIVE","batch_id":"upload-am-2026-07-13T08-40-05-671Z","ts":"2026-07-13T08:40:19.531Z"}
← bb0ac571 auto-save: 2026-07-13T00:51:43 (6 files) — pending-approval/
·
back to Designer Wallcoverings
·
auto-save: 2026-07-13T02:22:18 (8 files) — DW-Programming/Im 87598f1a →