jf_fabrics recrawl: survive HTTP 429 throttle (retry+backoff, concurrency 1, 1.1s spacing)

5f76fa07daebc0ead8b6d8657793bc336c452375 · 2026-06-10 16:35:52 -0700 · SteveStudio2
Files touched

M recrawl-jffabrics.js
Diff

commit 5f76fa07daebc0ead8b6d8657793bc336c452375
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date:   Wed Jun 10 16:35:52 2026 -0700

    jf_fabrics recrawl: survive HTTP 429 throttle (retry+backoff, concurrency 1, 1.1s spacing)
---
 recrawl-jffabrics.js | 41 ++++++++++++++++++++++++++++++++---------
 1 file changed, 32 insertions(+), 9 deletions(-)

diff --git a/recrawl-jffabrics.js b/recrawl-jffabrics.js
index 1ed381d..b125def 100644
--- a/recrawl-jffabrics.js
+++ b/recrawl-jffabrics.js
@@ -50,7 +50,14 @@ function httpGet(url, maxRedirects = 4) {
         res.resume();
         return httpGet(loc, maxRedirects - 1).then(resolve).catch(reject);
       }
-      if (res.statusCode !== 200) { res.resume(); return reject(new Error('HTTP ' + res.statusCode)); }
+      if (res.statusCode !== 200) {
+        res.resume();
+        const err = new Error('HTTP ' + res.statusCode);
+        err.statusCode = res.statusCode;
+        const ra = res.headers['retry-after'];
+        if (ra) { const s = parseInt(ra, 10); if (!isNaN(s)) err.retryAfterMs = Math.min(120000, s * 1000); }
+        return reject(err);
+      }
       let d = ''; res.on('data', c => d += c); res.on('end', () => resolve(d)); res.on('error', reject);
     });
     req2.on('error', reject);
@@ -59,14 +66,28 @@ function httpGet(url, maxRedirects = 4) {
 }
 const sleep = (ms) => new Promise(r => setTimeout(r, ms + Math.floor(Math.random() * 250)));
 
-// Fetch with retry on transient short pages (the site throttles bursts).
+// Fetch with retry on transient short pages AND on HTTP 429 rate-limiting.
+// JF Fabrics enforces an IP burst quota that depletes after ~25 quick hits, then
+// returns 429 until it refills — so on 429 we back off (Retry-After if given,
+// else escalating cooldown) and retry, letting the single worker self-pace to a
+// sustainable rate rather than burning the whole worklist into errors.
 async function fetchPage(url) {
-  for (let attempt = 1; attempt <= 3; attempt++) {
-    const html = await httpGet(url);
-    if (html && html.length >= MIN_BYTES) return html;
-    await sleep(1500 * attempt);
+  for (let attempt = 1; attempt <= 6; attempt++) {
+    try {
+      const html = await httpGet(url);
+      if (html && html.length >= MIN_BYTES) { await sleep(0); return html; }
+      await sleep(1500 * attempt); // short page (throttle/challenge) — back off + retry
+    } catch (e) {
+      if (e.statusCode === 429) {
+        const wait = e.retryAfterMs || Math.min(90000, 4000 * attempt * attempt); // 4s,16s,36s,64s,90s,90s
+        await sleep(wait);
+        continue;
+      }
+      if (attempt >= 6) throw e;
+      await sleep(1000 * attempt);
+    }
   }
-  throw new Error('short-page (throttled) after 3 tries');
+  throw new Error('exhausted retries (throttled)');
 }
 
 (async () => {
@@ -81,7 +102,9 @@ async function fetchPage(url) {
 
   let pages = 0, skus = 0, withImg = 0, withSpecs = 0, errs = 0;
   let idx = 0;
-  const concurrency = 3;
+  // Single worker — JF rate-limits an IP burst; serial + backoff is the only
+  // way the full worklist completes instead of mostly 429-erroring.
+  const concurrency = 1;
   async function worker() {
     while (idx < rows.length) {
       const i = idx++;
@@ -101,7 +124,7 @@ async function fetchPage(url) {
         }
         pages++;
         if (pages % 25 === 0) console.log(`  ${pages}/${rows.length} | ${skus} skus | ${withImg} w/img | ${withSpecs} w/specs | ${errs} errs`);
-        await sleep(400);
+        await sleep(1100); // polite spacing to stay under the burst quota
       } catch (e) {
         errs++;
         if (errs % 10 === 0) console.error(`  err ${url}: ${e.message}`);
← 32a7723 jf_fabrics all_images fix: full-res extractor + additive rec · back to Jf Fabrics Recrawl · DWJJ MFR/spec backfill: staged 63-product JF scrape + dry-ru 6909b88 →