← back to Dw Scrape Scheduler
auto-save: 2026-07-13T00:21:34 (4 files) — adapters/ build-manifest.js lib/ run-vendor.js
e183fed5ba53c3602d88cacff8344797b35116ab · 2026-07-13 00:21:37 -0700 · Steve Abrams
Files touched
A adapters/shopify.jsA build-manifest.jsA lib/db.jsA run-vendor.js
Diff
commit e183fed5ba53c3602d88cacff8344797b35116ab
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Jul 13 00:21:37 2026 -0700
auto-save: 2026-07-13T00:21:34 (4 files) — adapters/ build-manifest.js lib/ run-vendor.js
---
adapters/shopify.js | 51 ++++++++++++++++++++++++++++++++++++++++++++++
build-manifest.js | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++
lib/db.js | 16 +++++++++++++++
run-vendor.js | 36 +++++++++++++++++++++++++++++++++
4 files changed, 161 insertions(+)
diff --git a/adapters/shopify.js b/adapters/shopify.js
new file mode 100644
index 0000000..faac69a
--- /dev/null
+++ b/adapters/shopify.js
@@ -0,0 +1,51 @@
+// Shopify feed-first adapter — paginate <base>/products.json?limit=250&page=N.
+// $0 local HTTP, no login, no paid service. Retry-with-backoff on transient 503
+// (Shopify/CF blips — learned on Astek 2026-07-13). Returns normalized products.
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36';
+
+async function fetchPage(url, attempts = 4) {
+ let last = 0;
+ for (let i = 0; i < attempts; i++) {
+ const res = await fetch(url, { headers: { 'User-Agent': UA } });
+ if (res.ok) return res;
+ last = res.status;
+ if (i < attempts - 1) await new Promise(r => setTimeout(r, 1500 * (i + 1)));
+ }
+ throw new Error(`HTTP ${last} after ${attempts} attempts`);
+}
+
+// base = e.g. https://www.ellipopp.com (trailing slash tolerated)
+async function scrape(base, { maxPages = 60, politeMs = 150 } = {}) {
+ const root = base.replace(/\/+$/, '');
+ const feed = `${root}/products.json`;
+ const out = [];
+ for (let page = 1; page <= maxPages; page++) {
+ const res = await fetchPage(`${feed}?limit=250&page=${page}`);
+ const json = await res.json();
+ const products = json.products || [];
+ if (products.length === 0) break;
+ for (const p of products) {
+ const v0 = (p.variants || [])[0] || {};
+ const img0 = (p.images || [])[0] || {};
+ out.push({
+ mfr_sku: v0.sku || String(p.id),
+ handle: p.handle,
+ title: p.title,
+ product_type: p.product_type || '',
+ vendor: p.vendor || '',
+ price: v0.price != null ? Number(v0.price) : null,
+ body_html: p.body_html || '',
+ image_url: img0.src || null,
+ all_images: (p.images || []).map(i => i.src),
+ product_url: `${root}/products/${p.handle}`,
+ tags: typeof p.tags === 'string' ? p.tags.split(',').map(s => s.trim()).filter(Boolean) : (p.tags || []),
+ raw: p,
+ });
+ }
+ if (products.length < 250) break;
+ await new Promise(r => setTimeout(r, politeMs));
+ }
+ return out;
+}
+
+module.exports = { scrape, platform: 'shopify' };
diff --git a/build-manifest.js b/build-manifest.js
new file mode 100644
index 0000000..f032193
--- /dev/null
+++ b/build-manifest.js
@@ -0,0 +1,58 @@
+#!/usr/bin/env node
+// Generate data/manifest.json from vendor_registry — the single source of truth
+// for the staggered monthly scrape. TIER RULES (conservative — unsure => excluded):
+// TIER_A = feed-first: no creds, not skip_catalog_scrape, reachable platform we
+// have an adapter shape for (Shopify/WooCommerce/Magento).
+// headless = reachable-but-needs-render (Custom HTML/ASP.NET/TYPO3/Drupal/Next/Squarespace) => Tier-B (parked)
+// gated = has_credentials OR login_url OR paid (Browserbase/2captcha) => NEVER auto-scheduled
+// skip = Unreachable/Error/skip_catalog_scrape/archived
+// Assigns a stable day-of-month: honor existing crawl_cron day; else hash vendor_code -> 1..28.
+const fs = require('fs');
+const path = require('path');
+const { pool } = require('./lib/db');
+
+const FEED_PLATFORMS = { shopify: 'shopify', 'shopify 2.0': 'shopify', woocommerce: 'woo', wordpress: 'woo', magento: 'magento' };
+const HEADLESS = new Set(['custom html', 'asp.net', 'typo3', 'drupal', 'next.js', 'squarespace']);
+const DEAD = new Set(['unreachable', 'error', '?', '']);
+// Lines that must never be re-activated / are private-label-sensitive stay out of the auto scrape.
+const HARD_EXCLUDE = new Set(['nicolette_mayer']); // hard-archived (Steve rule)
+
+function dayFromCron(cron) {
+ if (!cron) return null;
+ const dom = String(cron).trim().split(/\s+/)[2];
+ const n = Number(dom);
+ return Number.isInteger(n) && n >= 1 && n <= 28 ? n : null;
+}
+function hashDay(code) { let h = 0; for (const c of code) h = (h * 31 + c.charCodeAt(0)) >>> 0; return (h % 28) + 1; }
+
+(async () => {
+ const { rows } = await pool.query(`
+ select vendor_code, vendor_name, lower(coalesce(website_tech_stack,'')) stack,
+ website_url, crawl_cron, coalesce(has_credentials,false) creds,
+ login_url, coalesce(skip_catalog_scrape,false) skipc
+ from vendor_registry
+ where website_url is not null and website_url <> ''
+ order by vendor_name`);
+
+ const tier_a = [], parked_headless = [], gated = [], skipped = [];
+ for (const r of rows) {
+ const entry = { vendor_code: r.vendor_code, vendor: r.vendor_name, platform: r.stack,
+ base_url: r.website_url, day: dayFromCron(r.crawl_cron) || hashDay(r.vendor_code) };
+ if (HARD_EXCLUDE.has(r.vendor_code)) { skipped.push({ ...entry, why: 'hard-archived' }); continue; }
+ if (r.creds || r.login_url) { gated.push({ ...entry, why: 'portal-login/creds' }); continue; }
+ if (r.skipc) { skipped.push({ ...entry, why: 'skip_catalog_scrape' }); continue; }
+ if (DEAD.has(r.stack)) { skipped.push({ ...entry, why: 'dead/unknown platform' }); continue; }
+ const adapter = FEED_PLATFORMS[r.stack];
+ if (adapter) { tier_a.push({ ...entry, adapter, enabled: true }); continue; }
+ if (HEADLESS.has(r.stack)) { parked_headless.push({ ...entry, adapter: 'headless', enabled: false }); continue; }
+ skipped.push({ ...entry, why: `no adapter for '${r.stack}'` });
+ }
+
+ const manifest = { generated_note: 'built from vendor_registry by build-manifest.js',
+ counts: { tier_a: tier_a.length, parked_headless: parked_headless.length, gated: gated.length, skipped: skipped.length },
+ tier_a, parked_headless, gated, skipped };
+ fs.writeFileSync(path.join(__dirname, 'data', 'manifest.json'), JSON.stringify(manifest, null, 2));
+ console.log('manifest.json written:', JSON.stringify(manifest.counts));
+ console.log('tier_a adapters:', tier_a.reduce((a, e) => { a[e.adapter] = (a[e.adapter] || 0) + 1; return a; }, {}));
+ await pool.end();
+})().catch(e => { console.error('FATAL', e); process.exit(1); });
diff --git a/lib/db.js b/lib/db.js
new file mode 100644
index 0000000..c29bd30
--- /dev/null
+++ b/lib/db.js
@@ -0,0 +1,16 @@
+// Shared pg pool. Reads DW_ADMIN_DB_PASSWORD from the secrets .env (same pattern
+// as scrape-astek.js) so no secret is hardcoded here.
+const fs = require('fs');
+const os = require('os');
+const { Pool } = require('pg');
+
+const PW = (() => {
+ try {
+ const env = fs.readFileSync(os.homedir() + '/Projects/secrets-manager/.env', 'utf8');
+ const m = env.match(/^DW_ADMIN_DB_PASSWORD=(.*)$/m);
+ return m ? m[1].replace(/^["']|["']$/g, '').trim() : (process.env.PGPASSWORD || '');
+ } catch { return process.env.PGPASSWORD || ''; }
+})();
+
+const pool = new Pool({ host: '127.0.0.1', port: 5432, user: 'dw_admin', database: 'dw_unified', password: PW });
+module.exports = { pool };
diff --git a/run-vendor.js b/run-vendor.js
new file mode 100644
index 0000000..e4468e0
--- /dev/null
+++ b/run-vendor.js
@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+// Scrape ONE vendor via its feed adapter -> snapshot to data/snapshots/.
+// PROOF STAGE: snapshot + count only, NO canonical dw_unified write yet (per-vendor
+// staging-table mapping is the next staged increment). Idempotent, $0 local.
+const fs = require('fs');
+const path = require('path');
+
+const ADAPTERS = { shopify: require('./adapters/shopify') };
+const SNAP_DIR = path.join(__dirname, 'data', 'snapshots');
+
+async function runVendor(entry) {
+ const adapter = ADAPTERS[entry.adapter];
+ if (!adapter) return { vendor: entry.vendor, ok: false, skipped: true, reason: `adapter '${entry.adapter}' not yet implemented` };
+ const t0 = Date.now();
+ try {
+ const products = await adapter.scrape(entry.base_url);
+ fs.mkdirSync(SNAP_DIR, { recursive: true });
+ const stamp = new Date().toISOString().slice(0, 10).replace(/-/g, '');
+ const file = path.join(SNAP_DIR, `${entry.vendor_code}-${stamp}.json`);
+ fs.writeFileSync(file, JSON.stringify({ vendor_code: entry.vendor_code, scraped_at: new Date().toISOString(), count: products.length, products }, null, 0));
+ return { vendor: entry.vendor, ok: true, count: products.length, ms: Date.now() - t0, file };
+ } catch (e) {
+ return { vendor: entry.vendor, ok: false, reason: e.message, ms: Date.now() - t0 };
+ }
+}
+
+module.exports = { runVendor };
+
+// CLI: node run-vendor.js <vendor_code> (looks the entry up in the manifest)
+if (require.main === module) {
+ const code = process.argv[2];
+ const manifest = JSON.parse(fs.readFileSync(path.join(__dirname, 'data', 'manifest.json'), 'utf8'));
+ const entry = [...manifest.tier_a, ...manifest.parked_headless].find(e => e.vendor_code === code);
+ if (!entry) { console.error(`vendor_code '${code}' not in manifest tier_a/parked_headless`); process.exit(1); }
+ runVendor(entry).then(r => { console.log(JSON.stringify(r)); process.exit(r.ok ? 0 : 1); });
+}
(oldest)
·
back to Dw Scrape Scheduler
·
scaffold dw-scrape-scheduler (DTD verdict A): manifest + sho a454b7e →