TK-10978 RebelWalls: focused 39-no-twin re-scrape — 31/39 vendor-404, 0 recoverable -> defer

4101e6dd514174a169e63cd5438864890da4e0c1 · 2026-09-02 12:06:48 -0700 · steve

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Files touched

A tk10978-rebelwalls/rescrape-39.mjs

Diff

commit 4101e6dd514174a169e63cd5438864890da4e0c1
Author: steve <steve@designerwallcoverings.com>
Date:   Wed Sep 2 12:06:48 2026 -0700

    TK-10978 RebelWalls: focused 39-no-twin re-scrape — 31/39 vendor-404, 0 recoverable -> defer
    
    Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 tk10978-rebelwalls/rescrape-39.mjs | 39 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/tk10978-rebelwalls/rescrape-39.mjs b/tk10978-rebelwalls/rescrape-39.mjs
new file mode 100644
index 0000000..627bcea
--- /dev/null
+++ b/tk10978-rebelwalls/rescrape-39.mjs
@@ -0,0 +1,39 @@
+#!/usr/bin/env node
+// TK-10978 — focused re-scrape of the 39 NO-TWIN candidates (READ-ONLY, $0).
+// Proper JSON-LD parse + 2s delay + 1 retry (avoids the soft-block the 494 sweep hit).
+// Classifies each: recoverable_live | oos | discontinued_410 | no_vendor_match.
+// Writes ONLY local artifacts. NO writes to Shopify or dw_unified. Makes nothing buyable.
+import { readFileSync, writeFileSync } from 'node:fs';
+import { execSync } from 'node:child_process';
+const HERE = new URL('.', import.meta.url).pathname;
+const rows = readFileSync(`${HERE}data/investigate-39-no-twin.csv`,'utf8').trim().split('\n').slice(1)
+  .map(l=>{const m=l.match(/^(\d+),(".*?"|[^,]*),(.*)$/);return m?{pid:m[1],title:m[2].replace(/^"|"$/g,''),sku:m[3]}:null;}).filter(Boolean);
+const slugify = t => t.replace(/ \| Rebel Walls\s*$/i,'').toLowerCase().replace(/['’]/g,'').replace(/[^a-z0-9]+/g,'-').replace(/^-+|-+$/g,'');
+const sh = s => `"${String(s??'').replace(/"/g,'""')}"`;
+async function fetchPage(slug){
+  const html = execSync(`curl -s -m 25 -w '\\n<<HTTP:%{http_code}>>' -A 'Mozilla/5.0 (compatible; DW-catalog/1.0)' "https://rebelwalls.com/${slug}"`,{encoding:'utf8',maxBuffer:16*1024*1024});
+  const http=(html.match(/<<HTTP:(\d+)>>/)||[])[1]||'000';
+  let ldsku='',price='',avail='';
+  const m=html.match(/<script type="application\/ld\+json">([\s\S]*?)<\/script>/);
+  if(m){try{const d=JSON.parse(m[1]);const it=(Array.isArray(d)?d:[d]).find(x=>x&&x['@type']==='Product');if(it){ldsku=it.sku||'';const off=it.offers||{};price=off.price??'';avail=String(off.availability||'').split('/').pop();}}catch{}}
+  return {http,ldsku,price,avail};
+}
+const results=[];
+for(const r of rows){
+  const slug=slugify(r.title);
+  let {http,ldsku,price,avail}=await fetchPage(slug);
+  if(http==='200'&&!ldsku){ execSync('sleep 2'); ({http,ldsku,price,avail}=await fetchPage(slug)); } // retry once
+  let cls;
+  if(http==='410')cls='discontinued_410';
+  else if(http==='200'&&ldsku&&Number(price)>10&&Number(price)<10000&&avail==='InStock')cls='recoverable_live';
+  else if(http==='200'&&ldsku&&avail&&avail!=='InStock')cls='oos';
+  else cls='no_vendor_match';
+  results.push({pid:r.pid,title:r.title,sku:r.sku,slug,http,ldsku,per_sqm:price,avail,cls});
+  execSync('sleep 2');
+}
+const cols=['pid','title','sku','slug','http','ldsku','per_sqm','avail','cls'];
+writeFileSync(`${HERE}data/rescrape-39.csv`,[cols.join(',')].concat(results.map(x=>cols.map(c=>sh(x[c])).join(','))).join('\n'));
+const by=results.reduce((a,x)=>{a[x.cls]=(a[x.cls]||0)+1;return a;},{});
+const recov=results.filter(x=>x.cls==='recoverable_live');
+writeFileSync(`${HERE}data/rescrape-39-summary.json`,JSON.stringify({ticket:'TK-10978',mode:'READ-ONLY',total:results.length,by_class:by,recoverable_live:recov.map(x=>({title:x.title,sku:x.sku,per_sqm:x.per_sqm}))},null,2));
+console.log(JSON.stringify({total:results.length,by_class:by,recoverable_live:recov.length},null,2));

← ae3731d TK-10978 RebelWalls: read-only recon + reversible archive ex · back to Dw Unbuyable Recovery Pilot · auto-data-snapshot: 2026-09-02T12:07:47 (8 data files) — tk1 cdd5b64 →