recrawl-cowtan: self-heal worklist WHERE re-selects pinterest/facebook-contaminated all_images

64d92ba8a0ba666bcd12f2b6e3c27e6e6fd40fb4 · 2026-06-10 17:28:43 -0700 · SteveStudio2

Files touched

M recrawl-cowtan.js

Diff

commit 64d92ba8a0ba666bcd12f2b6e3c27e6e6fd40fb4
Author: SteveStudio2 <steve@designerwallcoverings.com>
Date:   Wed Jun 10 17:28:43 2026 -0700

    recrawl-cowtan: self-heal worklist WHERE re-selects pinterest/facebook-contaminated all_images
---
 recrawl-cowtan.js | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/recrawl-cowtan.js b/recrawl-cowtan.js
index c59055e..35c7ab6 100644
--- a/recrawl-cowtan.js
+++ b/recrawl-cowtan.js
@@ -100,13 +100,17 @@ function specJson(entry) {
 (async () => {
   // worklist: rows still missing all_images. We crawl by distinct ProductCode (pattern)
   // since one Detail call returns every colourway of that pattern.
-  const where = `vendor_code = $1 AND (all_images IS NULL OR all_images = '' OR all_images = '[]' OR all_images = 'null')`;
+  // Self-heal clause: also re-select rows whose all_images was previously written with a
+  // 3rd-party share-link leak (pinterest/facebook), so a bad earlier run cures itself.
+  // Images are built host-anchored to cowtan's own CDN, so a clean re-run overwrites junk.
+  const contam = `all_images ILIKE '%pinterest%' OR all_images ILIKE '%facebook%'`;
+  const where = `vendor_code = $1 AND (all_images IS NULL OR all_images = '' OR all_images = '[]' OR all_images = 'null' OR ${contam})`;
   const q = `SELECT DISTINCT split_part(mfr_sku,'-',1) AS product_code, min(mfr_sku) AS seed_sku
              FROM ${TABLE} WHERE ${where} GROUP BY 1 ORDER BY 1` + (LIMIT ? ` LIMIT ${LIMIT}` : '');
   const { rows } = await pool.query(q, [VENDOR_CODE]).catch(async (e) => {
     // fleet per-vendor table has no vendor_code column — fall back to plain table scan.
     if (/vendor_code/.test(e.message)) {
-      const w2 = `(all_images IS NULL OR all_images='' OR all_images='[]' OR all_images='null')`;
+      const w2 = `(all_images IS NULL OR all_images='' OR all_images='[]' OR all_images='null' OR all_images ILIKE '%pinterest%' OR all_images ILIKE '%facebook%')`;
       return pool.query(`SELECT DISTINCT split_part(mfr_sku,'-',1) AS product_code, min(mfr_sku) AS seed_sku FROM ${TABLE} WHERE ${w2} GROUP BY 1 ORDER BY 1` + (LIMIT ? ` LIMIT ${LIMIT}` : ''));
     }
     throw e;

← c8b0219 scaffold: cowtan/osborne/graham_brown recrawl drivers + prob · back to Dw Cowtan Recrawl · (newest)