← back to Dw Scrape Scheduler
auto-save: 2026-07-13T00:51:43 (4 files) — data/manifest.json run-vendor.js adapters/woo.js verify-coverage.js
755fba7d314ba13b51cd50fa3c2345334ca1f8b0 · 2026-07-13 00:51:46 -0700 · Steve Abrams
Files touched
A adapters/woo.jsM data/manifest.jsonM run-vendor.jsA verify-coverage.js
Diff
commit 755fba7d314ba13b51cd50fa3c2345334ca1f8b0
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Jul 13 00:51:46 2026 -0700
auto-save: 2026-07-13T00:51:43 (4 files) — data/manifest.json run-vendor.js adapters/woo.js verify-coverage.js
---
adapters/woo.js | 61 ++++++++++++++++++++++++++++
data/manifest.json | 115 ++++++++++++++++++++++++++++++++++++++++-------------
run-vendor.js | 2 +-
verify-coverage.js | 33 +++++++++++++++
4 files changed, 183 insertions(+), 28 deletions(-)
diff --git a/adapters/woo.js b/adapters/woo.js
new file mode 100644
index 0000000..cb924c3
--- /dev/null
+++ b/adapters/woo.js
@@ -0,0 +1,61 @@
+// WooCommerce Store API adapter — public, no auth:
+// <base>/wp-json/wc/store/v1/products?per_page=100&page=N (fallback: .../store/products)
+// $0 local. Store API prices are in MINOR units (cents) — divide by 10^minor_unit.
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36';
+
+async function fetchJson(url, attempts = 4) {
+ let last = 0;
+ for (let i = 0; i < attempts; i++) {
+ const res = await fetch(url, { headers: { 'User-Agent': UA, 'Accept': 'application/json' } });
+ if (res.ok) return res.json();
+ last = res.status;
+ if (i < attempts - 1) await new Promise(r => setTimeout(r, 1500 * (i + 1)));
+ }
+ throw new Error(`HTTP ${last} after ${attempts} attempts`);
+}
+
+function priceOf(p) {
+ const pr = p.prices || {};
+ if (pr.price == null) return null;
+ const minor = Number(pr.currency_minor_unit ?? 2);
+ const n = Number(pr.price);
+ return Number.isFinite(n) ? n / Math.pow(10, minor) : null;
+}
+
+async function scrape(base, { maxPages = 60, politeMs = 200 } = {}) {
+ const root = base.replace(/\/+$/, '');
+ // Probe which endpoint shape this store exposes.
+ let path = null;
+ for (const cand of ['/wp-json/wc/store/v1/products', '/wp-json/wc/store/products']) {
+ try { const test = await fetchJson(`${root}${cand}?per_page=1`); if (Array.isArray(test)) { path = cand; break; } } catch { /* try next */ }
+ }
+ if (!path) throw new Error('no reachable WooCommerce Store API endpoint');
+
+ const out = [];
+ for (let page = 1; page <= maxPages; page++) {
+ const items = await fetchJson(`${root}${path}?per_page=100&page=${page}`);
+ if (!Array.isArray(items) || items.length === 0) break;
+ for (const p of items) {
+ const img0 = (p.images || [])[0] || {};
+ out.push({
+ mfr_sku: p.sku || String(p.id),
+ handle: p.slug || (p.permalink || '').split('/').filter(Boolean).pop(),
+ title: p.name,
+ product_type: (p.type || ''),
+ vendor: '',
+ price: priceOf(p),
+ body_html: p.description || p.short_description || '',
+ image_url: img0.src || null,
+ all_images: (p.images || []).map(i => i.src),
+ product_url: p.permalink || `${root}/?p=${p.id}`,
+ tags: (p.tags || []).map(t => t.name).filter(Boolean),
+ raw: p,
+ });
+ }
+ if (items.length < 100) break;
+ await new Promise(r => setTimeout(r, politeMs));
+ }
+ return out;
+}
+
+module.exports = { scrape, platform: 'woo' };
diff --git a/data/manifest.json b/data/manifest.json
index 74aafcb..fa18489 100644
--- a/data/manifest.json
+++ b/data/manifest.json
@@ -14,7 +14,8 @@
"base_url": "https://www.kravet.com/wallcovering?designer=Andrew%20Martin",
"day": 27,
"adapter": "magento",
- "enabled": true
+ "enabled": false,
+ "coverage": "adapter pending"
},
{
"vendor_code": "arte",
@@ -23,7 +24,8 @@
"base_url": "https://www.arte-international.com",
"day": 3,
"adapter": "magento",
- "enabled": true
+ "enabled": false,
+ "coverage": "adapter pending"
},
{
"vendor_code": "borastapeter",
@@ -32,7 +34,8 @@
"base_url": "https://www.borastapeter.com",
"day": 18,
"adapter": "magento",
- "enabled": true
+ "enabled": false,
+ "coverage": "adapter pending"
},
{
"vendor_code": "brand_mckenzie",
@@ -41,7 +44,8 @@
"base_url": "https://www.brandmckenzie.com",
"day": 20,
"adapter": "magento",
- "enabled": true
+ "enabled": false,
+ "coverage": "adapter pending"
},
{
"vendor_code": "carlisle",
@@ -50,7 +54,10 @@
"base_url": "https://www.carlisleco.com",
"day": 8,
"adapter": "shopify",
- "enabled": true
+ "enabled": true,
+ "coverage": "live",
+ "probe_count": 96,
+ "verified_at": "2026-07-13T07:49:31.760Z"
},
{
"vendor_code": "catchii",
@@ -59,7 +66,9 @@
"base_url": "https://www.catchii.com",
"day": 9,
"adapter": "woo",
- "enabled": true
+ "enabled": false,
+ "coverage": "no feed: no reachable WooCommerce Store API endpoint",
+ "verified_at": "2026-07-13T07:49:31.760Z"
},
{
"vendor_code": "colouranddesign",
@@ -68,7 +77,10 @@
"base_url": "https://colouranddesign.com",
"day": 22,
"adapter": "woo",
- "enabled": true
+ "enabled": true,
+ "coverage": "live",
+ "probe_count": 100,
+ "verified_at": "2026-07-13T07:49:31.760Z"
},
{
"vendor_code": "denovowall",
@@ -77,7 +89,10 @@
"base_url": "https://denovowall.com",
"day": 22,
"adapter": "woo",
- "enabled": true
+ "enabled": true,
+ "coverage": "live",
+ "probe_count": 100,
+ "verified_at": "2026-07-13T07:49:31.760Z"
},
{
"vendor_code": "ellipopp",
@@ -86,7 +101,10 @@
"base_url": "https://www.ellipopp.com",
"day": 20,
"adapter": "shopify",
- "enabled": true
+ "enabled": true,
+ "coverage": "live",
+ "probe_count": 250,
+ "verified_at": "2026-07-13T07:49:31.760Z"
},
{
"vendor_code": "farrow_ball",
@@ -95,7 +113,8 @@
"base_url": "https://www.farrow-ball.com",
"day": 8,
"adapter": "magento",
- "enabled": true
+ "enabled": false,
+ "coverage": "adapter pending"
},
{
"vendor_code": "flavor_paper",
@@ -104,7 +123,10 @@
"base_url": "https://www.flavorpaper.com",
"day": 13,
"adapter": "shopify",
- "enabled": true
+ "enabled": true,
+ "coverage": "live",
+ "probe_count": 250,
+ "verified_at": "2026-07-13T07:49:31.760Z"
},
{
"vendor_code": "gp_baker",
@@ -113,7 +135,9 @@
"base_url": "https://www.gpjbaker.com",
"day": 19,
"adapter": "woo",
- "enabled": true
+ "enabled": false,
+ "coverage": "no feed: no reachable WooCommerce Store API endpoint",
+ "verified_at": "2026-07-13T07:49:31.760Z"
},
{
"vendor_code": "GRD",
@@ -122,7 +146,10 @@
"base_url": "https://www.graduatecollection.co.uk",
"day": 2,
"adapter": "shopify",
- "enabled": true
+ "enabled": true,
+ "coverage": "live",
+ "probe_count": 104,
+ "verified_at": "2026-07-13T07:49:31.760Z"
},
{
"vendor_code": "jf_fabrics",
@@ -131,7 +158,9 @@
"base_url": "https://www.jffabrics.com",
"day": 13,
"adapter": "woo",
- "enabled": true
+ "enabled": false,
+ "coverage": "no feed: no reachable WooCommerce Store API endpoint",
+ "verified_at": "2026-07-13T07:49:31.760Z"
},
{
"vendor_code": "kamism",
@@ -140,7 +169,9 @@
"base_url": "https://kamism.co.jp/en/",
"day": 9,
"adapter": "woo",
- "enabled": true
+ "enabled": false,
+ "coverage": "no feed: no reachable WooCommerce Store API endpoint",
+ "verified_at": "2026-07-13T07:49:31.760Z"
},
{
"vendor_code": "lee_jofa",
@@ -149,7 +180,8 @@
"base_url": "https://www.leejofa.com",
"day": 28,
"adapter": "magento",
- "enabled": true
+ "enabled": false,
+ "coverage": "adapter pending"
},
{
"vendor_code": "lincrusta",
@@ -158,7 +190,10 @@
"base_url": "https://www.lincrusta.com",
"day": 3,
"adapter": "woo",
- "enabled": true
+ "enabled": true,
+ "coverage": "live",
+ "probe_count": 48,
+ "verified_at": "2026-07-13T07:49:31.760Z"
},
{
"vendor_code": "milton_king",
@@ -167,7 +202,10 @@
"base_url": "https://www.miltonandking.com",
"day": 13,
"adapter": "woo",
- "enabled": true
+ "enabled": true,
+ "coverage": "live",
+ "probe_count": 100,
+ "verified_at": "2026-07-13T07:49:31.760Z"
},
{
"vendor_code": "newmor",
@@ -176,7 +214,9 @@
"base_url": "https://www.newmor.com",
"day": 15,
"adapter": "woo",
- "enabled": true
+ "enabled": false,
+ "coverage": "no feed: no reachable WooCommerce Store API endpoint",
+ "verified_at": "2026-07-13T07:49:31.760Z"
},
{
"vendor_code": "newwall",
@@ -185,7 +225,10 @@
"base_url": "https://newwall.com",
"day": 23,
"adapter": "shopify",
- "enabled": true
+ "enabled": true,
+ "coverage": "live",
+ "probe_count": 250,
+ "verified_at": "2026-07-13T07:49:31.760Z"
},
{
"vendor_code": "nina_campbell",
@@ -194,7 +237,10 @@
"base_url": "https://www.ninacampbell.com",
"day": 21,
"adapter": "shopify",
- "enabled": true
+ "enabled": true,
+ "coverage": "live",
+ "probe_count": 250,
+ "verified_at": "2026-07-13T07:49:31.760Z"
},
{
"vendor_code": "nlxl",
@@ -203,7 +249,10 @@
"base_url": "https://nlxl.com",
"day": 26,
"adapter": "shopify",
- "enabled": true
+ "enabled": true,
+ "coverage": "live",
+ "probe_count": 198,
+ "verified_at": "2026-07-13T07:49:31.760Z"
},
{
"vendor_code": "sandberg",
@@ -212,7 +261,8 @@
"base_url": "https://sandbergwallpaper.com",
"day": 16,
"adapter": "magento",
- "enabled": true
+ "enabled": false,
+ "coverage": "adapter pending"
},
{
"vendor_code": "tres_tintas",
@@ -221,7 +271,10 @@
"base_url": "https://trestintas.com",
"day": 1,
"adapter": "woo",
- "enabled": true
+ "enabled": true,
+ "coverage": "live",
+ "probe_count": 100,
+ "verified_at": "2026-07-13T07:49:31.760Z"
},
{
"vendor_code": "wolf_gordon",
@@ -230,7 +283,8 @@
"base_url": "https://www.wolfgordon.com",
"day": 23,
"adapter": "magento",
- "enabled": true
+ "enabled": false,
+ "coverage": "adapter pending"
},
{
"vendor_code": "zoffany",
@@ -239,7 +293,8 @@
"base_url": "https://www.zoffany.com",
"day": 16,
"adapter": "magento",
- "enabled": true
+ "enabled": false,
+ "coverage": "adapter pending"
}
],
"parked_headless": [
@@ -1028,5 +1083,11 @@
"day": 1,
"why": "skip_catalog_scrape"
}
- ]
+ ],
+ "coverage": {
+ "live": 12,
+ "dead": 5,
+ "pending": 9,
+ "verified_at": "2026-07-13T07:49:31.760Z"
+ }
}
\ No newline at end of file
diff --git a/run-vendor.js b/run-vendor.js
index 88e7ee2..a671cb4 100644
--- a/run-vendor.js
+++ b/run-vendor.js
@@ -5,7 +5,7 @@
const fs = require('fs');
const path = require('path');
-const ADAPTERS = { shopify: require('./adapters/shopify') };
+const ADAPTERS = { shopify: require('./adapters/shopify'), woo: require('./adapters/woo') };
const { writeStaging } = require('./lib/write-staging');
const SNAP_DIR = path.join(__dirname, 'data', 'snapshots');
diff --git a/verify-coverage.js b/verify-coverage.js
new file mode 100644
index 0000000..650f657
--- /dev/null
+++ b/verify-coverage.js
@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+// Probe each Tier-A vendor's adapter (1 page only) and set enabled=true/false from
+// real evidence, so the scheduled dispatcher never blindly retries a dead feed.
+// Vendors whose adapter isn't implemented yet (e.g. magento) => enabled=false,
+// note 'adapter pending'. Rewrites data/manifest.json in place.
+const fs = require('fs');
+const path = require('path');
+
+const ADAPTERS = { shopify: require('./adapters/shopify'), woo: require('./adapters/woo') };
+const MANIFEST = path.join(__dirname, 'data', 'manifest.json');
+const PROBE_TIMEOUT = 25000;
+
+const withTimeout = (p, ms) => Promise.race([p, new Promise((_, rej) => setTimeout(() => rej(new Error('probe timeout')), ms))]);
+
+(async () => {
+ const m = JSON.parse(fs.readFileSync(MANIFEST, 'utf8'));
+ const nowIso = new Date().toISOString();
+ let live = 0, dead = 0, pending = 0;
+ for (const e of m.tier_a) {
+ const adapter = ADAPTERS[e.adapter];
+ if (!adapter) { e.enabled = false; e.coverage = 'adapter pending'; pending++; continue; }
+ try {
+ const products = await withTimeout(adapter.scrape(e.base_url, { maxPages: 1 }), PROBE_TIMEOUT);
+ if (products.length > 0) { e.enabled = true; e.coverage = 'live'; e.probe_count = products.length; live++; }
+ else { e.enabled = false; e.coverage = 'empty feed'; dead++; }
+ } catch (err) { e.enabled = false; e.coverage = `no feed: ${err.message}`; dead++; }
+ e.verified_at = nowIso;
+ process.stdout.write(` ${e.enabled ? '✓' : '✗'} ${e.vendor} [${e.adapter}] ${e.coverage}${e.probe_count ? ' ('+e.probe_count+'/pg)' : ''}\n`);
+ }
+ m.coverage = { live, dead, pending, verified_at: nowIso };
+ fs.writeFileSync(MANIFEST, JSON.stringify(m, null, 2));
+ console.log(`\ncoverage: ${live} live, ${dead} dead/empty, ${pending} adapter-pending (of ${m.tier_a.length} tier_a)`);
+})().catch(e => { console.error('FATAL', e); process.exit(1); });
← 1b2ac91 wire canonical scrape_staging DB writes (dedupe + insert/upd
·
back to Dw Scrape Scheduler
·
add daily dispatcher launchd wrapper + plist (staggered mont ec519a3 →