Add flag-discontinued.js: HEAD-probe net-new image-less rows, mark hard-404/410 as sync_status=discontinued (reversible via discontinued_prev_status)

ba1e429943f9668a8b40e12336febf018f29eae3 · 2026-06-10 20:56:43 -0700 · SteveStudio2
Files touched

A flag-discontinued.js
Diff

commit ba1e429943f9668a8b40e12336febf018f29eae3
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date:   Wed Jun 10 20:56:43 2026 -0700

    Add flag-discontinued.js: HEAD-probe net-new image-less rows, mark hard-404/410 as sync_status=discontinued (reversible via discontinued_prev_status)
---
 flag-discontinued.js | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/flag-discontinued.js b/flag-discontinued.js
new file mode 100644
index 0000000..1fe67b1
--- /dev/null
+++ b/flag-discontinued.js
@@ -0,0 +1,61 @@
+#!/usr/bin/env node
+// flag-discontinued.js — probe net-new rows missing all_images; mark dead (404/410) URLs
+// as discontinued. ADDITIVE + REVERSIBLE: backs up prior sync_status to discontinued_prev_status.
+// Usage: node flag-discontinued.js <vendor_code> [--limit N] [--dry-run] [--apply]
+require('dotenv').config({ path: '../dw-price-stock/.env' });
+const { Pool } = require('pg');
+const VENDOR = process.argv[2];
+const DRY = process.argv.includes('--dry-run') || !process.argv.includes('--apply');
+const LIM = (() => { const i = process.argv.indexOf('--limit'); return i>-1?parseInt(process.argv[i+1],10):0; })();
+if (!VENDOR) { console.error('need vendor_code'); process.exit(1); }
+const pool = new Pool({ connectionString: process.env.DATABASE_URL });
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36';
+const CONC = 6;
+
+async function head(url){
+  try {
+    const ctrl = new AbortController(); const t = setTimeout(()=>ctrl.abort(), 12000);
+    let r = await fetch(url, { method:'HEAD', redirect:'follow', signal:ctrl.signal, headers:{'User-Agent':UA} });
+    // some servers 405 on HEAD -> retry GET
+    if (r.status===405 || r.status===403) r = await fetch(url, { method:'GET', redirect:'follow', signal:ctrl.signal, headers:{'User-Agent':UA} });
+    clearTimeout(t); return r.status;
+  } catch(e){ return 0; } // 0 = network error (don't flag — could be transient)
+}
+
+(async () => {
+  const { rows } = await pool.query(
+    `SELECT mfr_sku, product_url FROM vendor_catalog
+     WHERE vendor_code=$1
+       AND (all_images IS NULL OR btrim(all_images::text) IN ('','[]','null','{}'))
+       AND COALESCE(on_shopify,false)=false AND shopify_product_id IS NULL
+       AND product_url IS NOT NULL AND product_url <> ''
+       AND COALESCE(sync_status,'') NOT IN ('discontinued','never_push','archived_duplicate')
+     ${LIM?`LIMIT ${LIM}`:''}`, [VENDOR]);
+  console.log(`${VENDOR}: probing ${rows.length} net-new image-less URLs (DRY=${DRY})`);
+  let dead=[], live=0, neterr=0, i=0;
+  async function worker(){
+    while (i < rows.length){
+      const row = rows[i++];
+      const st = await head(row.product_url);
+      if (st===404 || st===410) dead.push(row.mfr_sku);
+      else if (st===0) neterr++;
+      else live++;
+      if ((dead.length+live+neterr) % 100 === 0) console.log(`  ${dead.length+live+neterr}/${rows.length} | dead ${dead.length} live ${live} neterr ${neterr}`);
+    }
+  }
+  await Promise.all(Array.from({length:CONC}, worker));
+  console.log(`RESULT ${VENDOR}: dead=${dead.length} live=${live} neterr=${neterr} of ${rows.length}`);
+  if (!DRY && dead.length){
+    // chunked update with backup of prior status
+    for (let j=0;j<dead.length;j+=500){
+      const chunk = dead.slice(j,j+500);
+      await pool.query(
+        `UPDATE vendor_catalog SET
+            discontinued_prev_status = COALESCE(discontinued_prev_status, sync_status),
+            sync_status='discontinued', updated_at=NOW()
+         WHERE vendor_code=$1 AND mfr_sku = ANY($2)`, [VENDOR, chunk]);
+    }
+    console.log(`  APPLIED: ${dead.length} rows -> sync_status='discontinued' (prev backed up to discontinued_prev_status)`);
+  }
+  await pool.end();
+})();
← 127175f recrawl-schumacher: dry-run skips DB write + self-heal pinte · back to Dw Vendor Scrapers Local · (newest)