Fix: send User-Agent on sitemap fetch (WG 403s UA-less requests) — restores full catalog discovery

946e786b1b7d643b83964ad656495e56dc06dc8d · 2026-06-19 09:11:16 -0700 · Steve

Files touched

M scrape-wolfgordon.js

Diff

commit 946e786b1b7d643b83964ad656495e56dc06dc8d
Author: Steve <steve@designerwallcoverings.com>
Date:   Fri Jun 19 09:11:16 2026 -0700

    Fix: send User-Agent on sitemap fetch (WG 403s UA-less requests) — restores full catalog discovery
---
 scrape-wolfgordon.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scrape-wolfgordon.js b/scrape-wolfgordon.js
index 2bbcb64..c7ef252 100644
--- a/scrape-wolfgordon.js
+++ b/scrape-wolfgordon.js
@@ -50,7 +50,11 @@ function fetchPage(url, maxRedirects = 5) {
 
 function fetchXml(url) {
   return new Promise((resolve, reject) => {
-    https.get(url, { timeout: 15000 }, (res) => {
+    // WG returns 403 to UA-less requests — a UA is mandatory on the sitemaps too.
+    https.get(url, { timeout: 15000, headers: {
+      'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+      'Accept': 'application/xml,text/xml,*/*;q=0.8',
+    } }, (res) => {
       if (res.statusCode === 301 || res.statusCode === 302) {
         return fetchXml(res.headers.location).then(resolve).catch(reject);
       }

← 6235ec2 Wolf Gordon Mac2-local crawl: parser updated for current Cra · back to Wolfgordon Crawl · Add WG re-onboarding reconciliation (dry-run, normalized-SKU 9f1964c →