harden astek scraper: retry feed pages with backoff on transient 503 (Shopify/CF blip); browser UA

a682f9552eeb5ebbc1c75e9b9aeeb93c73d08f74 · 2026-07-12 23:58:30 -0700 · Steve

Files touched

M scripts/scrape-astek.js

Diff

commit a682f9552eeb5ebbc1c75e9b9aeeb93c73d08f74
Author: Steve <steve@designerwallcoverings.com>
Date:   Sun Jul 12 23:58:30 2026 -0700

    harden astek scraper: retry feed pages with backoff on transient 503 (Shopify/CF blip); browser UA
---
 scripts/scrape-astek.js | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/scripts/scrape-astek.js b/scripts/scrape-astek.js
index 4b9e218..42d5d69 100644
--- a/scripts/scrape-astek.js
+++ b/scripts/scrape-astek.js
@@ -55,12 +55,27 @@ function parseSpecs(html) {
   };
 }
 
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36';
+
+// Shopify/CF occasionally returns a transient 503 on the products.json feed
+// (seen 2026-07-13). Retry with backoff so the monthly unattended job doesn't
+// die on a single blip. Raises only after all attempts are exhausted.
+async function fetchFeedPage(url, attempts = 4) {
+  let lastStatus = 0;
+  for (let i = 0; i < attempts; i++) {
+    const res = await fetch(url, { headers: { 'User-Agent': UA } });
+    if (res.ok) return res;
+    lastStatus = res.status;
+    if (i < attempts - 1) await new Promise(r => setTimeout(r, 1500 * (i + 1))); // 1.5s, 3s, 4.5s
+  }
+  throw new Error(`feed HTTP ${lastStatus} after ${attempts} attempts`);
+}
+
 async function fetchAllProducts() {
   let page = 1, all = [];
   while (true) {
     const url = `${FEED}?limit=250&page=${page}`;
-    const res = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0 DW-internal-catalog' } });
-    if (!res.ok) throw new Error(`feed HTTP ${res.status} on page ${page}`);
+    const res = await fetchFeedPage(url);
     const json = await res.json();
     const products = json.products || [];
     if (products.length === 0) break;

← 9de635e auto-save: 2026-07-12T23:51:20 (1 files) — data/products.jso · back to Astek Landing · data: astek refresh 2026-07-13 (6187 variants re-scraped, sn e5c3452 →