← back to 1970swallpaper
pull-1970s: browser UA, check statusCode, retry 429 with backoff
a4f2550ce8b9d898391124cb97d6086ce81905c7 · 2026-09-10 11:32:06 -0700 · Steve Abrams
Files touched
Diff
commit a4f2550ce8b9d898391124cb97d6086ce81905c7
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 10 11:32:06 2026 -0700
pull-1970s: browser UA, check statusCode, retry 429 with backoff
---
scripts/pull-1970s.js | 23 +++++++++++++++++++++--
1 file changed, 21 insertions(+), 2 deletions(-)
diff --git a/scripts/pull-1970s.js b/scripts/pull-1970s.js
index 4f70b88..a01bb1f 100644
--- a/scripts/pull-1970s.js
+++ b/scripts/pull-1970s.js
@@ -13,8 +13,9 @@ const REJECT_TITLE = /(visual.{0,3}merchandiser|bh.?90210|beverly.?hills.?90210|
function fetchPage(page) {
return new Promise((resolve, reject) => {
https.get(`https://designerwallcoverings.com/products.json?limit=250&page=${page}`, {
- headers: { 'User-Agent': 'Mozilla/5.0 1970swallpaper-builder' }
+ headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36' }
}, (res) => {
+ if (res.statusCode !== 200) { res.resume(); const e = new Error('HTTP ' + res.statusCode + ' from products.json page ' + page); e.statusCode = res.statusCode; return reject(e); }
let data = '';
res.on('data', c => data += c);
res.on('end', () => { try { resolve(JSON.parse(data).products || []); } catch(e) { reject(e); } });
@@ -22,6 +23,23 @@ function fetchPage(page) {
});
}
+// The upstream rate-limits per User-Agent string. This script used to fire up to
+// 36 pages back-to-back (x9 sites from refresh-all.sh) with no delay, trip a 429,
+// and -- because it never checked statusCode -- turn the HTML error page into a
+// JSON.parse failure. Retry with backoff, and pace the page loop.
+const sleep = ms => new Promise(r => setTimeout(r, ms));
+async function fetchPageRetry(page, tries = 5) {
+ for (let i = 0; i < tries; i++) {
+ try { return await fetchPage(page); }
+ catch (e) {
+ if (e.statusCode !== 429 || i === tries - 1) throw e;
+ const wait = 2000 * Math.pow(2, i);
+ console.log(`[retry] page ${page} got 429, waiting ${wait}ms (attempt ${i + 2}/${tries})`);
+ await sleep(wait);
+ }
+ }
+}
+
function feel(tags, title) {
const t = (tags || []).map(x => x.toLowerCase()).join(' ');
const tt = (title || '').toLowerCase();
@@ -37,11 +55,12 @@ function feel(tags, title) {
(async () => {
const all = [];
for (let page = 1; page <= 35; page++) {
- const products = await fetchPage(page);
+ const products = await fetchPageRetry(page);
if (!products.length) break;
all.push(...products);
process.stdout.write(`page ${page}: ${products.length} (total ${all.length})\n`);
if (products.length < 250) break;
+ await sleep(600); // pace: stay under the per-UA rate limit
}
console.log(`fetched ${all.length}`);
← f456289 Standardize privacy policy to /privacy.html + footer link (A
·
back to 1970swallpaper
·
auto-data-snapshot: 2026-09-10T11:43:04 (1 data files) — dat 3bd00dd →