← back to Dw Scrape Scheduler
scaffold dw-scrape-scheduler (DTD verdict A): manifest + shopify adapter + dispatcher; shopify cohort proven (/bin/zsh local)
a454b7ed4369790b3a420c691be068ed1227a1ca · 2026-07-13 00:23:47 -0700 · Steve Abrams
Files touched
A .gitignoreA README.mdA data/manifest.jsonA dispatcher.jsA package-lock.jsonA package.json
Diff
commit a454b7ed4369790b3a420c691be068ed1227a1ca
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Jul 13 00:23:47 2026 -0700
scaffold dw-scrape-scheduler (DTD verdict A): manifest + shopify adapter + dispatcher; shopify cohort proven (/bin/zsh local)
---
.gitignore | 6 +
README.md | 12 +
data/manifest.json | 1032 ++++++++++++++++++++++++++++++++++++++++++++++++++++
dispatcher.js | 52 +++
package-lock.json | 161 ++++++++
package.json | 1 +
6 files changed, 1264 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..136dbc0
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,6 @@
+node_modules/
+.env*
+data/snapshots/
+logs/
+*.log
+.DS_Store
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..e27db46
--- /dev/null
+++ b/README.md
@@ -0,0 +1,12 @@
+# dw-scrape-scheduler
+
+Staggered monthly feed-first vendor scrape. DTD verdict A (2026-07-13): one generic
+feed-first runner + a single daily launchd dispatcher that runs only the Tier-A vendors
+whose crawl_cron day == today, concurrency cap 2, local-models-only, no Gemini.
+
+- `build-manifest.js` — vendor_registry -> data/manifest.json (tiers by platform+creds)
+- `adapters/` — feed adapters (shopify LIVE; woo/magento TODO)
+- `run-vendor.js` — scrape one vendor -> data/snapshots/ (PROOF stage: snapshot+count, no canonical write yet)
+- `dispatcher.js` — daily selector + concurrency cap (--day/--vendor/--all/--dry-run/--cap)
+
+Tier-C (portal-login/paid-Browserbase) is NEVER auto-scheduled (gated).
diff --git a/data/manifest.json b/data/manifest.json
new file mode 100644
index 0000000..74aafcb
--- /dev/null
+++ b/data/manifest.json
@@ -0,0 +1,1032 @@
+{
+ "generated_note": "built from vendor_registry by build-manifest.js",
+ "counts": {
+ "tier_a": 26,
+ "parked_headless": 13,
+ "gated": 33,
+ "skipped": 50
+ },
+ "tier_a": [
+ {
+ "vendor_code": "andrew_martin",
+ "vendor": "Andrew Martin",
+ "platform": "magento",
+ "base_url": "https://www.kravet.com/wallcovering?designer=Andrew%20Martin",
+ "day": 27,
+ "adapter": "magento",
+ "enabled": true
+ },
+ {
+ "vendor_code": "arte",
+ "vendor": "Arte International",
+ "platform": "magento",
+ "base_url": "https://www.arte-international.com",
+ "day": 3,
+ "adapter": "magento",
+ "enabled": true
+ },
+ {
+ "vendor_code": "borastapeter",
+ "vendor": "Borastapeter",
+ "platform": "magento",
+ "base_url": "https://www.borastapeter.com",
+ "day": 18,
+ "adapter": "magento",
+ "enabled": true
+ },
+ {
+ "vendor_code": "brand_mckenzie",
+ "vendor": "Brand McKenzie",
+ "platform": "magento",
+ "base_url": "https://www.brandmckenzie.com",
+ "day": 20,
+ "adapter": "magento",
+ "enabled": true
+ },
+ {
+ "vendor_code": "carlisle",
+ "vendor": "Carlisle & Co",
+ "platform": "shopify 2.0",
+ "base_url": "https://www.carlisleco.com",
+ "day": 8,
+ "adapter": "shopify",
+ "enabled": true
+ },
+ {
+ "vendor_code": "catchii",
+ "vendor": "Catchii",
+ "platform": "wordpress",
+ "base_url": "https://www.catchii.com",
+ "day": 9,
+ "adapter": "woo",
+ "enabled": true
+ },
+ {
+ "vendor_code": "colouranddesign",
+ "vendor": "Colour & Design",
+ "platform": "wordpress",
+ "base_url": "https://colouranddesign.com",
+ "day": 22,
+ "adapter": "woo",
+ "enabled": true
+ },
+ {
+ "vendor_code": "denovowall",
+ "vendor": "DeNovo Wall",
+ "platform": "wordpress",
+ "base_url": "https://denovowall.com",
+ "day": 22,
+ "adapter": "woo",
+ "enabled": true
+ },
+ {
+ "vendor_code": "ellipopp",
+ "vendor": "Elli Popp",
+ "platform": "shopify",
+ "base_url": "https://www.ellipopp.com",
+ "day": 20,
+ "adapter": "shopify",
+ "enabled": true
+ },
+ {
+ "vendor_code": "farrow_ball",
+ "vendor": "Farrow & Ball",
+ "platform": "magento",
+ "base_url": "https://www.farrow-ball.com",
+ "day": 8,
+ "adapter": "magento",
+ "enabled": true
+ },
+ {
+ "vendor_code": "flavor_paper",
+ "vendor": "Flavor Paper",
+ "platform": "shopify 2.0",
+ "base_url": "https://www.flavorpaper.com",
+ "day": 13,
+ "adapter": "shopify",
+ "enabled": true
+ },
+ {
+ "vendor_code": "gp_baker",
+ "vendor": "GP & J Baker",
+ "platform": "wordpress",
+ "base_url": "https://www.gpjbaker.com",
+ "day": 19,
+ "adapter": "woo",
+ "enabled": true
+ },
+ {
+ "vendor_code": "GRD",
+ "vendor": "Graduate Collection",
+ "platform": "shopify",
+ "base_url": "https://www.graduatecollection.co.uk",
+ "day": 2,
+ "adapter": "shopify",
+ "enabled": true
+ },
+ {
+ "vendor_code": "jf_fabrics",
+ "vendor": "JF Fabrics",
+ "platform": "wordpress",
+ "base_url": "https://www.jffabrics.com",
+ "day": 13,
+ "adapter": "woo",
+ "enabled": true
+ },
+ {
+ "vendor_code": "kamism",
+ "vendor": "Kamism",
+ "platform": "wordpress",
+ "base_url": "https://kamism.co.jp/en/",
+ "day": 9,
+ "adapter": "woo",
+ "enabled": true
+ },
+ {
+ "vendor_code": "lee_jofa",
+ "vendor": "Lee Jofa",
+ "platform": "magento",
+ "base_url": "https://www.leejofa.com",
+ "day": 28,
+ "adapter": "magento",
+ "enabled": true
+ },
+ {
+ "vendor_code": "lincrusta",
+ "vendor": "Lincrusta",
+ "platform": "wordpress",
+ "base_url": "https://www.lincrusta.com",
+ "day": 3,
+ "adapter": "woo",
+ "enabled": true
+ },
+ {
+ "vendor_code": "milton_king",
+ "vendor": "Milton & King",
+ "platform": "wordpress",
+ "base_url": "https://www.miltonandking.com",
+ "day": 13,
+ "adapter": "woo",
+ "enabled": true
+ },
+ {
+ "vendor_code": "newmor",
+ "vendor": "Newmor",
+ "platform": "wordpress",
+ "base_url": "https://www.newmor.com",
+ "day": 15,
+ "adapter": "woo",
+ "enabled": true
+ },
+ {
+ "vendor_code": "newwall",
+ "vendor": "Newwall",
+ "platform": "shopify",
+ "base_url": "https://newwall.com",
+ "day": 23,
+ "adapter": "shopify",
+ "enabled": true
+ },
+ {
+ "vendor_code": "nina_campbell",
+ "vendor": "Nina Campbell",
+ "platform": "shopify 2.0",
+ "base_url": "https://www.ninacampbell.com",
+ "day": 21,
+ "adapter": "shopify",
+ "enabled": true
+ },
+ {
+ "vendor_code": "nlxl",
+ "vendor": "NLXL",
+ "platform": "shopify 2.0",
+ "base_url": "https://nlxl.com",
+ "day": 26,
+ "adapter": "shopify",
+ "enabled": true
+ },
+ {
+ "vendor_code": "sandberg",
+ "vendor": "Sandberg",
+ "platform": "magento",
+ "base_url": "https://sandbergwallpaper.com",
+ "day": 16,
+ "adapter": "magento",
+ "enabled": true
+ },
+ {
+ "vendor_code": "tres_tintas",
+ "vendor": "Tres Tintas",
+ "platform": "wordpress",
+ "base_url": "https://trestintas.com",
+ "day": 1,
+ "adapter": "woo",
+ "enabled": true
+ },
+ {
+ "vendor_code": "wolf_gordon",
+ "vendor": "Wolf Gordon",
+ "platform": "magento",
+ "base_url": "https://www.wolfgordon.com",
+ "day": 23,
+ "adapter": "magento",
+ "enabled": true
+ },
+ {
+ "vendor_code": "zoffany",
+ "vendor": "Zoffany",
+ "platform": "magento",
+ "base_url": "https://www.zoffany.com",
+ "day": 16,
+ "adapter": "magento",
+ "enabled": true
+ }
+ ],
+ "parked_headless": [
+ {
+ "vendor_code": "atomic50",
+ "vendor": "Atomic 50 Ceilings",
+ "platform": "squarespace",
+ "base_url": "https://www.atomic50ceilings.com",
+ "day": 11,
+ "adapter": "headless",
+ "enabled": false
+ },
+ {
+ "vendor_code": "black_edition",
+ "vendor": "Black Edition",
+ "platform": "custom html",
+ "base_url": "https://www.blackedition.com",
+ "day": 19,
+ "adapter": "headless",
+ "enabled": false
+ },
+ {
+ "vendor_code": "holly_hunt",
+ "vendor": "Holly Hunt",
+ "platform": "drupal",
+ "base_url": "https://www.hollyhunt.com",
+ "day": 9,
+ "adapter": "headless",
+ "enabled": false
+ },
+ {
+ "vendor_code": "kirkby_design",
+ "vendor": "Kirkby Design",
+ "platform": "custom html",
+ "base_url": "https://www.kirkbydesign.com",
+ "day": 4,
+ "adapter": "headless",
+ "enabled": false
+ },
+ {
+ "vendor_code": "manuel_canovas",
+ "vendor": "Manuel Canovas",
+ "platform": "asp.net",
+ "base_url": "https://www.manuelcanovas.com",
+ "day": 2,
+ "adapter": "headless",
+ "enabled": false
+ },
+ {
+ "vendor_code": "mark_alexander",
+ "vendor": "Mark Alexander",
+ "platform": "custom html",
+ "base_url": "https://www.markalexander.com",
+ "day": 5,
+ "adapter": "headless",
+ "enabled": false
+ },
+ {
+ "vendor_code": "naturale54",
+ "vendor": "Naturale 54",
+ "platform": "asp.net",
+ "base_url": "https://naturale54.com",
+ "day": 4,
+ "adapter": "headless",
+ "enabled": false
+ },
+ {
+ "vendor_code": "nobilis",
+ "vendor": "Nobilis",
+ "platform": "custom html",
+ "base_url": "https://www.nobilis.fr",
+ "day": 14,
+ "adapter": "headless",
+ "enabled": false
+ },
+ {
+ "vendor_code": "pierre_frey",
+ "vendor": "Pierre Frey",
+ "platform": "custom html",
+ "base_url": "https://www.pierrefrey.com",
+ "day": 15,
+ "adapter": "headless",
+ "enabled": false
+ },
+ {
+ "vendor_code": "versa_ds",
+ "vendor": "Versa Designed Surfaces",
+ "platform": "typo3",
+ "base_url": "https://www.versadesignedsurfaces.com",
+ "day": 6,
+ "adapter": "headless",
+ "enabled": false
+ },
+ {
+ "vendor_code": "villa_nova",
+ "vendor": "Villa Nova",
+ "platform": "custom html",
+ "base_url": "https://www.villanova.co.uk",
+ "day": 26,
+ "adapter": "headless",
+ "enabled": false
+ },
+ {
+ "vendor_code": "zimmer_rohde",
+ "vendor": "Zimmer + Rohde",
+ "platform": "typo3",
+ "base_url": "https://www.zimmer-rohde.com",
+ "day": 18,
+ "adapter": "headless",
+ "enabled": false
+ },
+ {
+ "vendor_code": "zinc_textile",
+ "vendor": "Zinc Textile",
+ "platform": "custom html",
+ "base_url": "https://www.zinctextile.com",
+ "day": 27,
+ "adapter": "headless",
+ "enabled": false
+ }
+ ],
+ "gated": [
+ {
+ "vendor_code": "w1838",
+ "vendor": "1838 Wallcoverings",
+ "platform": "shopify 2.0",
+ "base_url": "https://1838wallcoverings.com",
+ "day": 10,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "arteriors",
+ "vendor": "Arteriors",
+ "platform": "unreachable",
+ "base_url": "https://www.arteriorshome.com",
+ "day": 17,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "bnwalls",
+ "vendor": "BN Walls",
+ "platform": "cloudflare cdn",
+ "base_url": "https://bnwalls.com",
+ "day": 11,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "brewster_york",
+ "vendor": "Brewster & York",
+ "platform": "unreachable",
+ "base_url": "https://www.yorkwall.com",
+ "day": 3,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "contrado",
+ "vendor": "Contrado",
+ "platform": "magento",
+ "base_url": "https://www.contrado.com",
+ "day": 16,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "dedar",
+ "vendor": "Dedar",
+ "platform": "wordpress",
+ "base_url": "https://dedar.com",
+ "day": 9,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "designers_guild",
+ "vendor": "Designers Guild",
+ "platform": "magento",
+ "base_url": "https://www.designersguild.com",
+ "day": 15,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "designtex",
+ "vendor": "Designtex",
+ "platform": "wordpress",
+ "base_url": "https://www.designtex.com",
+ "day": 9,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "fabricut",
+ "vendor": "Fabricut",
+ "platform": "magento",
+ "base_url": "https://fabricut.com",
+ "day": 5,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "folia",
+ "vendor": "Folia Fabrics",
+ "platform": "unreachable",
+ "base_url": "https://www.folia-fabrics.com",
+ "day": 18,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "graham_brown",
+ "vendor": "Graham & Brown",
+ "platform": "next.js",
+ "base_url": "https://www.grahambrown.com",
+ "day": 8,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "hygge",
+ "vendor": "Hygge & West",
+ "platform": "unreachable",
+ "base_url": "https://www.hyggeandwest.com",
+ "day": 12,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "innovations",
+ "vendor": "Innovations",
+ "platform": "magento",
+ "base_url": "https://innovationsusa.com",
+ "day": 6,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "knoll",
+ "vendor": "Knoll",
+ "platform": "cloudflare cdn",
+ "base_url": "https://www.knoll.com",
+ "day": 10,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "koroseal",
+ "vendor": "Koroseal",
+ "platform": "magento",
+ "base_url": "https://koroseal.com",
+ "day": 2,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "kravet",
+ "vendor": "Kravet",
+ "platform": "magento",
+ "base_url": "https://www.kravet.com",
+ "day": 1,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "malibu",
+ "vendor": "Malibu Wallpaper",
+ "platform": "unreachable",
+ "base_url": "https://www.malibuwallpaper.com",
+ "day": 14,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "marburg",
+ "vendor": "Marburg",
+ "platform": "wordpress",
+ "base_url": "https://www.marburg.com",
+ "day": 2,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "maya",
+ "vendor": "Maya Romanoff",
+ "platform": "custom html",
+ "base_url": "https://mayaromanoff.com",
+ "day": 7,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "mindthegap",
+ "vendor": "Mind the Gap",
+ "platform": "magento",
+ "base_url": "https://mindtheg.com",
+ "day": 12,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "muralsource",
+ "vendor": "Mural Source",
+ "platform": "magento",
+ "base_url": "https://themuralsource.com",
+ "day": 16,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "osborne",
+ "vendor": "Osborne & Little",
+ "platform": "wordpress",
+ "base_url": "https://www.osborneandlittle.com",
+ "day": 11,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "phillip_jeffries",
+ "vendor": "Phillip Jeffries",
+ "platform": "magento",
+ "base_url": "https://phillipjeffries.com",
+ "day": 6,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "ralph_lauren",
+ "vendor": "Ralph Lauren",
+ "platform": "custom html",
+ "base_url": "https://www.ralphlaurenhome.com",
+ "day": 8,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "rebel_walls",
+ "vendor": "Rebel Walls",
+ "platform": "magento",
+ "base_url": "https://rebelwalls.com",
+ "day": 9,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "romo",
+ "vendor": "Romo",
+ "platform": "custom html",
+ "base_url": "https://romo.com",
+ "day": 5,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "schumacher",
+ "vendor": "Schumacher",
+ "platform": "next.js",
+ "base_url": "https://www.schumacher.com",
+ "day": 1,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "spoonflower",
+ "vendor": "Spoonflower",
+ "platform": "unreachable",
+ "base_url": "https://www.spoonflower.com",
+ "day": 18,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "stout",
+ "vendor": "Stout Textiles",
+ "platform": "magento",
+ "base_url": "https://www.stouttextiles.com",
+ "day": 15,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "thibaut",
+ "vendor": "Thibaut",
+ "platform": "shopify",
+ "base_url": "https://www.thibautdesign.com",
+ "day": 2,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "timorous",
+ "vendor": "Timorous Beasties",
+ "platform": "php",
+ "base_url": "https://www.timorousbeasties.com",
+ "day": 17,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "wallquest",
+ "vendor": "WallQuest",
+ "platform": "magento",
+ "base_url": "https://www.wallquest.com",
+ "day": 15,
+ "why": "portal-login/creds"
+ },
+ {
+ "vendor_code": "york_contract",
+ "vendor": "York Contract",
+ "platform": "asp.net",
+ "base_url": "https://www.yorkwall.com",
+ "day": 26,
+ "why": "portal-login/creds"
+ }
+ ],
+ "skipped": [
+ {
+ "vendor_code": "anna_french",
+ "vendor": "Anna French",
+ "platform": "error",
+ "base_url": "https://www.thibautdesign.com/anna-french",
+ "day": 10,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "armani_casa",
+ "vendor": "Armani Casa",
+ "platform": "unreachable",
+ "base_url": "https://www.armanicasa.com",
+ "day": 17,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "baker_lifestyle",
+ "vendor": "Baker Lifestyle",
+ "platform": "unreachable",
+ "base_url": "https://www.bakerlifestyle.com",
+ "day": 28,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "british_walls",
+ "vendor": "British Walls",
+ "platform": "unreachable",
+ "base_url": "https://www.britishwalls.com",
+ "day": 11,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "brunschwig",
+ "vendor": "Brunschwig & Fils",
+ "platform": "unreachable",
+ "base_url": "https://www.brunschwig.com",
+ "day": 23,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "china_seas",
+ "vendor": "China Seas",
+ "platform": "unreachable",
+ "base_url": "https://www.chinaseas.com",
+ "day": 12,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "clarke_clarke",
+ "vendor": "Clarke And Clarke",
+ "platform": "unreachable",
+ "base_url": "https://www.clarke-clarke.com",
+ "day": 20,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "cole_son",
+ "vendor": "Cole & Son",
+ "platform": "unreachable",
+ "base_url": "https://www.cole-and-son.com",
+ "day": 4,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "colefax_fowler",
+ "vendor": "Colefax & Fowler",
+ "platform": "unreachable",
+ "base_url": "https://www.colefaxfowler.com",
+ "day": 24,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "coordonne",
+ "vendor": "Coordonné",
+ "platform": "unreachable",
+ "base_url": "https://coordonne.com",
+ "day": 11,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "cowtan_tout",
+ "vendor": "Cowtan & Tout",
+ "platform": "unreachable",
+ "base_url": "https://www.cowtantout.com",
+ "day": 8,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "desima_european_textiles",
+ "vendor": "Desima European Textiles",
+ "platform": "",
+ "base_url": "https://www.desima.com",
+ "day": 3,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "dolce_gabbana",
+ "vendor": "Dolce & Gabbana",
+ "platform": "unreachable",
+ "base_url": "https://www.dolcegabbana.com",
+ "day": 5,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "donghia",
+ "vendor": "Donghia",
+ "platform": "unreachable",
+ "base_url": "https://www.donghia.com",
+ "day": 6,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "DWDP",
+ "vendor": "Dupenny",
+ "platform": "unreachable",
+ "base_url": "https://dupenny.com",
+ "day": 20,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "bespoke",
+ "vendor": "DW Bespoke Studio",
+ "platform": "unreachable",
+ "base_url": "https://bfrdesignstudio.com",
+ "day": 18,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "elitis",
+ "vendor": "Elitis",
+ "platform": "unreachable",
+ "base_url": "https://www.elitis.fr",
+ "day": 4,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "et_cie",
+ "vendor": "Et Cie",
+ "platform": "unreachable",
+ "base_url": "https://www.etcie.com",
+ "day": 7,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "fentucci",
+ "vendor": "Fentucci",
+ "platform": "error",
+ "base_url": "https://www.fentucci.com",
+ "day": 22,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "gaston_daniela",
+ "vendor": "Gaston Y Daniela",
+ "platform": "error",
+ "base_url": "https://www.gastonydaniela.com",
+ "day": 25,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "gimmersta",
+ "vendor": "Gimmersta",
+ "platform": "wordpress",
+ "base_url": "https://gimmersta.com",
+ "day": 26,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "glass_bead",
+ "vendor": "Glass Bead (DW House Brand - Sourced)",
+ "platform": "unreachable",
+ "base_url": "https://www.glassbead.com",
+ "day": 9,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "glitter_walls",
+ "vendor": "Glitter Walls (multi-source)",
+ "platform": "custom html",
+ "base_url": "https://www.glitterwalls.com",
+ "day": 12,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "harlequin",
+ "vendor": "Harlequin",
+ "platform": "unreachable",
+ "base_url": "https://www.harlequin.uk.com",
+ "day": 4,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "hermes",
+ "vendor": "Hermes",
+ "platform": "error",
+ "base_url": "https://www.hermes.com",
+ "day": 10,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "hollywood",
+ "vendor": "Hollywood Wallcoverings",
+ "platform": "unreachable",
+ "base_url": "https://www.hollywood-ep.com",
+ "day": 8,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "iksel",
+ "vendor": "IKSEL",
+ "platform": "wordpress",
+ "base_url": "https://www.iksel.com",
+ "day": 11,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "la_walls",
+ "vendor": "LA Walls",
+ "platform": "magento",
+ "base_url": "https://www.lawalls.com",
+ "day": 7,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "laura_ashley",
+ "vendor": "Laura Ashley",
+ "platform": "shopify 2.0",
+ "base_url": "https://www.lauraashley.com",
+ "day": 12,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "leed_walls",
+ "vendor": "LEED Walls",
+ "platform": "unreachable",
+ "base_url": "https://www.leedwalls.com",
+ "day": 23,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "missoni",
+ "vendor": "Missoni",
+ "platform": "error",
+ "base_url": "https://www.missoni.com",
+ "day": 14,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "mulberry",
+ "vendor": "Mulberry",
+ "platform": "unreachable",
+ "base_url": "https://www.mulberryhome.com",
+ "day": 16,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "nicolette_mayer",
+ "vendor": "Nicolette Mayer",
+ "platform": "shopify 2.0",
+ "base_url": "https://www.nicolettemayer.com",
+ "day": 28,
+ "why": "hard-archived"
+ },
+ {
+ "vendor_code": "novasuede",
+ "vendor": "Novasuede",
+ "platform": "custom",
+ "base_url": "https://www.novasuede.com",
+ "day": 21,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "paul_montgomery",
+ "vendor": "Paul Montgomery",
+ "platform": "unreachable",
+ "base_url": "https://www.paulmontgomery.com",
+ "day": 17,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "primo_leathers",
+ "vendor": "Primo Leathers",
+ "platform": "unreachable",
+ "base_url": "https://www.primoleathers.com",
+ "day": 22,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "relativity_textiles",
+ "vendor": "Relativity Textiles",
+ "platform": "unreachable",
+ "base_url": "https://www.retrowalls.com",
+ "day": 3,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "roberto_cavalli",
+ "vendor": "Roberto Cavalli",
+ "platform": "unreachable",
+ "base_url": "https://www.robertocavalli.com",
+ "day": 27,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "sancar",
+ "vendor": "Sancar",
+ "platform": "unreachable",
+ "base_url": "https://sancarwallcoverings.com",
+ "day": 1,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "scalamandre",
+ "vendor": "Scalamandre",
+ "platform": "unreachable",
+ "base_url": "https://www.scalamandre.com",
+ "day": 4,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "studio_printworks",
+ "vendor": "Studio Printworks",
+ "platform": "unreachable",
+ "base_url": "https://studioprintworks.com",
+ "day": 3,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "surface_stick",
+ "vendor": "Surface Stick",
+ "platform": "unreachable",
+ "base_url": "https://www.surfacestick.com",
+ "day": 6,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "tecnografica",
+ "vendor": "Tecnografica",
+ "platform": "",
+ "base_url": "https://www.tecnografica.net",
+ "day": 9,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "threads",
+ "vendor": "Threads",
+ "platform": "unreachable",
+ "base_url": "https://www.threadsfabrics.com",
+ "day": 25,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "twil_karin",
+ "vendor": "TWIL Karin",
+ "platform": "wordpress",
+ "base_url": "https://www.mdcwall.com",
+ "day": 13,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "ultrasuede",
+ "vendor": "Ultrasuede",
+ "platform": "php",
+ "base_url": "https://www.ultrasuede.com",
+ "day": 24,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "versace",
+ "vendor": "Versace",
+ "platform": "error",
+ "base_url": "https://www.versace.com",
+ "day": 1,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "william_morris",
+ "vendor": "William Morris",
+ "platform": "error",
+ "base_url": "https://www.williammorris.com",
+ "day": 19,
+ "why": "dead/unknown platform"
+ },
+ {
+ "vendor_code": "jeffrey_stevens",
+ "vendor": "York (Jeffrey Stevens source)",
+ "platform": "unreachable",
+ "base_url": "https://www.jeffreystevens.com",
+ "day": 10,
+ "why": "skip_catalog_scrape"
+ },
+ {
+ "vendor_code": "phillipe_romano",
+ "vendor": "York/Brewster (Phillipe Romano source)",
+ "platform": "unreachable",
+ "base_url": "https://www.philliperomano.com",
+ "day": 1,
+ "why": "skip_catalog_scrape"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/dispatcher.js b/dispatcher.js
new file mode 100644
index 0000000..d03f473
--- /dev/null
+++ b/dispatcher.js
@@ -0,0 +1,52 @@
+#!/usr/bin/env node
+// Daily dispatcher — the "don't overload" heart. Runs ONLY the Tier-A vendors
+// whose assigned day-of-month == today, at a hard concurrency cap. Installed as a
+// single daily launchd job; because each vendor carries a distinct day, only ~1-2
+// vendors run on any given day. Local-models-only: feed adapters need no model;
+// Gemini enrich-ai-tags is NOT invoked here.
+//
+// Flags: --day N (override today) | --vendor CODE (one) | --all (ignore day) | --dry-run | --cap N
+const fs = require('fs');
+const path = require('path');
+const { runVendor } = require('./run-vendor');
+
+const args = process.argv.slice(2);
+const flag = (n, d) => { const i = args.indexOf(n); return i >= 0 ? (args[i + 1] && !args[i + 1].startsWith('--') ? args[i + 1] : true) : d; };
+const CAP = Number(flag('--cap', 2));
+const DRY = args.includes('--dry-run');
+const ALL = args.includes('--all');
+const ONE = flag('--vendor', null);
+const DAY = Number(flag('--day', new Date().getDate()));
+
+async function pool(items, cap, fn) {
+ const out = []; let i = 0;
+ const workers = Array.from({ length: Math.min(cap, items.length) }, async () => {
+ while (i < items.length) { const idx = i++; out[idx] = await fn(items[idx]); }
+ });
+ await Promise.all(workers);
+ return out;
+}
+
+(async () => {
+ const manifest = JSON.parse(fs.readFileSync(path.join(__dirname, 'data', 'manifest.json'), 'utf8'));
+ let todo = manifest.tier_a.filter(e => e.enabled);
+ if (ONE) todo = todo.filter(e => e.vendor_code === ONE);
+ else if (!ALL) todo = todo.filter(e => e.day === DAY);
+
+ const stamp = new Date().toISOString().slice(0, 10);
+ const log = path.join(__dirname, 'logs', `dispatch-${stamp}.log`);
+ fs.mkdirSync(path.dirname(log), { recursive: true });
+ const line = s => { fs.appendFileSync(log, s + '\n'); console.log(s); };
+
+ line(`[${new Date().toISOString()}] dispatch day=${DAY} cap=${CAP} selected=${todo.length}${DRY ? ' (DRY)' : ''}${ALL ? ' (ALL)' : ''}`);
+ if (todo.length === 0) { line(' nothing scheduled today — done.'); return; }
+ if (DRY) { todo.forEach(e => line(` would scrape: ${e.vendor} [${e.adapter}] ${e.base_url}`)); return; }
+
+ const results = await pool(todo, CAP, runVendor);
+ let ok = 0, fail = 0;
+ for (const r of results) {
+ if (r.ok) { ok++; line(` ✓ ${r.vendor}: ${r.count} products (${r.ms}ms)`); }
+ else { fail++; line(` ✗ ${r.vendor}: ${r.skipped ? 'SKIP ' : 'FAIL '}${r.reason}`); }
+ }
+ line(`[${new Date().toISOString()}] done: ${ok} ok, ${fail} fail/skip`);
+})().catch(e => { console.error('FATAL', e); process.exit(1); });
diff --git a/package-lock.json b/package-lock.json
new file mode 100644
index 0000000..8facd05
--- /dev/null
+++ b/package-lock.json
@@ -0,0 +1,161 @@
+{
+ "name": "dw-scrape-scheduler",
+ "version": "0.1.0",
+ "lockfileVersion": 3,
+ "requires": true,
+ "packages": {
+ "": {
+ "name": "dw-scrape-scheduler",
+ "version": "0.1.0",
+ "dependencies": {
+ "pg": "^8.22.0"
+ }
+ },
+ "node_modules/pg": {
+ "version": "8.22.0",
+ "resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz",
+ "integrity": "sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==",
+ "license": "MIT",
+ "dependencies": {
+ "pg-connection-string": "^2.14.0",
+ "pg-pool": "^3.14.0",
+ "pg-protocol": "^1.15.0",
+ "pg-types": "2.2.0",
+ "pgpass": "1.0.5"
+ },
+ "engines": {
+ "node": ">= 16.0.0"
+ },
+ "optionalDependencies": {
+ "pg-cloudflare": "^1.4.0"
+ },
+ "peerDependencies": {
+ "pg-native": ">=3.0.1"
+ },
+ "peerDependenciesMeta": {
+ "pg-native": {
+ "optional": true
+ }
+ }
+ },
+ "node_modules/pg-cloudflare": {
+ "version": "1.4.0",
+ "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz",
+ "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==",
+ "license": "MIT",
+ "optional": true
+ },
+ "node_modules/pg-connection-string": {
+ "version": "2.14.0",
+ "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.14.0.tgz",
+ "integrity": "sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==",
+ "license": "MIT"
+ },
+ "node_modules/pg-int8": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
+ "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
+ "license": "ISC",
+ "engines": {
+ "node": ">=4.0.0"
+ }
+ },
+ "node_modules/pg-pool": {
+ "version": "3.14.0",
+ "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz",
+ "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==",
+ "license": "MIT",
+ "peerDependencies": {
+ "pg": ">=8.0"
+ }
+ },
+ "node_modules/pg-protocol": {
+ "version": "1.15.0",
+ "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.15.0.tgz",
+ "integrity": "sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==",
+ "license": "MIT"
+ },
+ "node_modules/pg-types": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
+ "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
+ "license": "MIT",
+ "dependencies": {
+ "pg-int8": "1.0.1",
+ "postgres-array": "~2.0.0",
+ "postgres-bytea": "~1.0.0",
+ "postgres-date": "~1.0.4",
+ "postgres-interval": "^1.1.0"
+ },
+ "engines": {
+ "node": ">=4"
+ }
+ },
+ "node_modules/pgpass": {
+ "version": "1.0.5",
+ "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
+ "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
+ "license": "MIT",
+ "dependencies": {
+ "split2": "^4.1.0"
+ }
+ },
+ "node_modules/postgres-array": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
+ "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=4"
+ }
+ },
+ "node_modules/postgres-bytea": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz",
+ "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/postgres-date": {
+ "version": "1.0.7",
+ "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
+ "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/postgres-interval": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
+ "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
+ "license": "MIT",
+ "dependencies": {
+ "xtend": "^4.0.0"
+ },
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/split2": {
+ "version": "4.2.0",
+ "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
+ "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
+ "license": "ISC",
+ "engines": {
+ "node": ">= 10.x"
+ }
+ },
+ "node_modules/xtend": {
+ "version": "4.0.2",
+ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+ "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=0.4"
+ }
+ }
+ }
+}
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..c91b673
--- /dev/null
+++ b/package.json
@@ -0,0 +1 @@
+{"name":"dw-scrape-scheduler","version":"0.1.0","private":true,"description":"Staggered monthly feed-first vendor scrape (DTD verdict A, 2026-07-13)","dependencies":{"pg":"^8.22.0"}}
\ No newline at end of file
← e183fed auto-save: 2026-07-13T00:21:34 (4 files) — adapters/ build-m
·
back to Dw Scrape Scheduler
·
wire canonical scrape_staging DB writes (dedupe + insert/upd 1b2ac91 →