← back to Rebel Walls Scraper

Add resumable checkpointed crawl mode for full 4309-product run

93cb660f0c2a0564c272015b8333a4c8248bcfdb · 2026-06-03 20:56:15 -0700 · SteveStudio2

Files touched

• M scripts/scrape.js

Diff

commit 93cb660f0c2a0564c272015b8333a4c8248bcfdb
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date:   Wed Jun 3 20:56:15 2026 -0700

    Add resumable checkpointed crawl mode for full 4309-product run
---
 scripts/scrape.js | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 63 insertions(+), 1 deletion(-)

diff --git a/scripts/scrape.js b/scripts/scrape.js
index 2c80e59..80cdaeb 100644
--- a/scripts/scrape.js
+++ b/scripts/scrape.js
@@ -4,6 +4,7 @@
  * Modes:
  *   node scripts/scrape.js enumerate            -> data/product-urls.json
  *   node scripts/scrape.js pilot <N> [offset]   -> output/pilot-<date>.json
+ *   node scripts/scrape.js crawl                -> output/crawl-<date>.json (full set, resumable, checkpointed)
  *   node scripts/scrape.js fullproduct <json>   -> data-quality scorecard
  *
  * Zero-dependency: uses global fetch (Node 18+). Polite: 1.5s delay, single-threaded.
@@ -213,6 +214,66 @@ async function pilot(n, offset) {
   return outFile;
 }
 
+/* ---- crawl: full set, single-threaded, checkpointed + resumable ---- */
+async function crawl() {
+  const urlsPath = path.join(DATA, 'product-urls.json');
+  let urls;
+  if (fs.existsSync(urlsPath)) urls = JSON.parse(fs.readFileSync(urlsPath, 'utf8'));
+  else urls = await enumerate();
+
+  fs.mkdirSync(OUT, { recursive: true });
+  const outFile = path.join(OUT, `crawl-${today()}.json`);
+
+  // Resume support: if outFile exists, skip URLs already captured OK.
+  let results = [];
+  const done = new Set();
+  if (fs.existsSync(outFile)) {
+    try {
+      results = JSON.parse(fs.readFileSync(outFile, 'utf8'));
+      for (const r of results) if (r && r.product_url && !r.error) done.add(r.product_url);
+      console.error(`Resume: ${done.size} already captured in ${path.basename(outFile)}; ${results.length} total rows.`);
+    } catch (_) { results = []; }
+  }
+  // Drop prior error rows for URLs we'll retry, then re-add fresh.
+  results = results.filter((r) => r && r.product_url && !r.error);
+
+  const todo = urls.filter((u) => !done.has(u));
+  console.error(`Crawling ${todo.length} of ${urls.length} URLs (${done.size} already done). ~${Math.round(todo.length * DELAY_MS / 1000 / 60)} min @ ${DELAY_MS}ms.`);
+
+  let ok = 0, err = 0;
+  for (let i = 0; i < todo.length; i++) {
+    const u = todo[i];
+    process.stderr.write(`[${i + 1}/${todo.length}] ${u} … `);
+    let attempt = 0, captured = null, lastErr = null;
+    while (attempt < 3 && !captured) {
+      attempt++;
+      try {
+        captured = await parseProduct(u);
+      } catch (e) {
+        lastErr = e;
+        if (attempt < 3) { process.stderr.write(`retry${attempt} `); await sleep(DELAY_MS * 2); }
+      }
+    }
+    if (captured) {
+      results.push(captured);
+      ok++;
+      process.stderr.write(`OK ${captured.mfr_sku} $${captured.price_retail}/m²\n`);
+    } else {
+      results.push({ product_url: u, error: String((lastErr && lastErr.message) || lastErr) });
+      err++;
+      process.stderr.write(`ERR ${(lastErr && lastErr.message) || lastErr}\n`);
+    }
+    // Checkpoint every 25 products (and on the last one) so a crash loses ≤25.
+    if ((i + 1) % 25 === 0 || i === todo.length - 1) {
+      fs.writeFileSync(outFile, JSON.stringify(results, null, 2));
+    }
+    if (i < todo.length - 1) await sleep(DELAY_MS);
+  }
+  fs.writeFileSync(outFile, JSON.stringify(results, null, 2));
+  console.error(`\nCrawl complete: ${ok} OK, ${err} ERR this run; ${results.length} total rows -> ${outFile}`);
+  return outFile;
+}
+
 /* ---- fullproduct gate ---- */
 function fullproduct(jsonPath) {
   const rows = JSON.parse(fs.readFileSync(jsonPath, 'utf8')).filter((r) => !r.error);
@@ -260,9 +321,10 @@ function fullproduct(jsonPath) {
   try {
     if (mode === 'enumerate') await enumerate();
     else if (mode === 'pilot') await pilot(parseInt(a || '25', 10), parseInt(b || '0', 10));
+    else if (mode === 'crawl') await crawl();
     else if (mode === 'fullproduct') fullproduct(a);
     else {
-      console.error('Usage: scrape.js enumerate | pilot <N> [offset] | fullproduct <json>');
+      console.error('Usage: scrape.js enumerate | pilot <N> [offset] | crawl | fullproduct <json>');
       process.exit(1);
     }
   } catch (e) {

← f8fa280 Rebel Walls scraper: enumerate 4309 + 25-SKU pilot (fullprod  ·  back to Rebel Walls Scraper  ·  Add dw_unified import script: gate + 2.5x markup + DWRW SKU 87c6420 →