DTD Option C: generic product_url refresher (fetch HTML free -> local exo/ollama extract $0 -> update staging, no publish). Wired into resolver -> resolvable vendors 1->162. Proven end-to-end on Thibaut T19631 (COASTLINE grasscloth extracted via exo in 1.1s). Hard-timeout + redirect-cap fetcher; hand-wired dirs stay priority, skip is last resort.

0dc05260511c9b2b8c99c6bdab008914e60c430c · 2026-07-07 06:43:36 -0700 · steve
Files touched

M lib/config.js
A lib/generic-url-refresh.js
M lib/resolve-pipeline.js
A tools/fix-slack-webhook.js
Diff

commit 0dc05260511c9b2b8c99c6bdab008914e60c430c
Author: steve <steve@designerwallcoverings.com>
Date:   Tue Jul 7 06:43:36 2026 -0700

    DTD Option C: generic product_url refresher (fetch HTML free -> local exo/ollama extract $0 -> update staging, no publish). Wired into resolver -> resolvable vendors 1->162. Proven end-to-end on Thibaut T19631 (COASTLINE grasscloth extracted via exo in 1.1s). Hard-timeout + redirect-cap fetcher; hand-wired dirs stay priority, skip is last resort.
---
 lib/config.js              |   2 +
 lib/generic-url-refresh.js | 131 +++++++++++++++++++++++++++++++++++++++++++++
 lib/resolve-pipeline.js    |  19 +++++--
 tools/fix-slack-webhook.js |  33 ++++++++++++
 4 files changed, 182 insertions(+), 3 deletions(-)

diff --git a/lib/config.js b/lib/config.js
index ec1fdb2..66bd49e 100644
--- a/lib/config.js
+++ b/lib/config.js
@@ -32,6 +32,8 @@ module.exports = {
   EXCLUDE_VENDOR_CODES: (process.env.RECRAWL_EXCLUDE || 'wallquest').split(',').map(s => s.trim()).filter(Boolean),
 
   // --- pipeline resolution ---
+  // Max SKUs the generic product_url refresher re-fetches per vendor per night.
+  GENERIC_LIMIT: parseInt(process.env.RECRAWL_GENERIC_LIMIT || '25', 10),
   DW_ROOT: path.join(HOME, 'Projects', 'Designer-Wallcoverings'),
   REFRESH_SCRIPTS_DIR: path.join(HOME, 'Projects', 'Designer-Wallcoverings', 'scripts'),
 
diff --git a/lib/generic-url-refresh.js b/lib/generic-url-refresh.js
new file mode 100644
index 0000000..06b1f4a
--- /dev/null
+++ b/lib/generic-url-refresh.js
@@ -0,0 +1,131 @@
+'use strict';
+// GENERIC product_url refresher (DTD Option C). For each existing SKU with a
+// product_url, fetch the page (free), extract price/specs via the LOCAL enrich-router
+// (exo -> Ollama, $0, no paid API), and UPDATE dw_unified staging. NO publish.
+//
+// Covers any vendor catalog that has a product_url column populated — hundreds of
+// vendors — instead of hand-wiring each. Hand-wired <code>-refresh scripts remain the
+// fallback for vendors with no product_url / anti-bot walls / JS-rendered pages.
+//
+// Safe: --dry-run fetches + extracts but writes NOTHING. Live writes only price/spec
+// columns that exist on the table, only when extraction is confident.
+const https = require('https');
+const { pool } = require('./db');
+const enrich = require('./enrich-router');
+const cfg = require('./config');
+
+function fetchHtml(url, timeoutMs = 15000, hops = 0) {
+  return new Promise((resolve) => {
+    if (hops > 4) return resolve({ ok: false, status: 'too-many-redirects' });
+    let done = false;
+    const finish = (v) => { if (done) return; done = true; try { req.destroy(); } catch {} resolve(v); };
+    // GUARANTEED hard deadline — one tarpitting/anti-bot URL can never stall the batch.
+    const hard = setTimeout(() => finish({ ok: false, status: 'timeout' }), timeoutMs);
+    let req;
+    try {
+      req = https.get(url, { headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh) DW-recrawl/1.0' } }, (res) => {
+        if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+          res.resume(); clearTimeout(hard); if (done) return; done = true;
+          return resolve(fetchHtml(new URL(res.headers.location, url).href, timeoutMs, hops + 1));
+        }
+        if (res.statusCode !== 200) { res.resume(); clearTimeout(hard); return finish({ ok: false, status: res.statusCode }); }
+        let body = ''; res.setEncoding('utf8');
+        res.on('data', d => { body += d; if (body.length > 800_000) { res.destroy(); } });
+        res.on('end', () => { clearTimeout(hard); finish({ ok: true, html: body }); });
+      });
+      req.on('error', () => { clearTimeout(hard); finish({ ok: false, status: 'error' }); });
+    } catch { clearTimeout(hard); finish({ ok: false, status: 'bad-url' }); }
+  });
+}
+
+// Strip HTML to visible-ish text, trimmed to a model-friendly size.
+function htmlToText(html) {
+  return html
+    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
+    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&')
+    .replace(/\s+/g, ' ')
+    .trim()
+    .slice(0, 6000);
+}
+
+// Ask the LOCAL model to extract structured fields from the page text. Returns object or null.
+async function extractFields(vendorName, sku, text) {
+  const prompt =
+    `You are extracting wallcovering product data from a vendor product page.\n` +
+    `Vendor: ${vendorName}. SKU: ${sku}.\n` +
+    `From the PAGE TEXT below, return ONLY a compact JSON object with any of these keys you can find ` +
+    `(omit keys you can't find, no prose): {"price": number, "width_in": number, "repeat_in": number, ` +
+    `"material": string, "pattern_name": string, "discontinued": boolean}.\n\nPAGE TEXT:\n${text}`;
+  const res = await enrich.chat([{ role: 'user', content: prompt }]);
+  if (!res || res.skipped || !res.text) return null;
+  const m = res.text.match(/\{[\s\S]*\}/);
+  if (!m) return null;
+  try { return { provider: res.provider, fields: JSON.parse(m[0]) }; } catch { return null; }
+}
+
+// Which price/spec columns actually exist on this table (only update real columns).
+async function tableColumns(table) {
+  const { rows } = await pool.query(
+    `SELECT column_name FROM information_schema.columns WHERE table_name = $1`, [table]);
+  return new Set(rows.map(r => r.column_name));
+}
+
+// Refresh up to `limit` stale SKUs (with product_url) for one vendor catalog.
+// opts: { table, vendorName, skuCol, limit, dryRun }
+async function refreshVendor({ table, vendorName, skuCol = 'mfr_sku', limit = 25, dryRun = true }) {
+  const cols = await tableColumns(table);
+  if (!cols.has('product_url')) return { table, skipped: 'no product_url column' };
+  const orderStale = cols.has('updated_at') ? 'ORDER BY updated_at ASC NULLS FIRST' : '';
+  const { rows } = await pool.query(
+    `SELECT ${skuCol} AS sku, product_url FROM ${table}
+      WHERE product_url IS NOT NULL AND product_url <> '' ${orderStale} LIMIT $1`, [limit]);
+
+  let fetched = 0, extracted = 0, updated = 0, failed = 0;
+  const sample = [];
+  for (const r of rows) {
+    const page = await fetchHtml(r.product_url);
+    if (!page.ok) { failed++; continue; }
+    fetched++;
+    const ex = await extractFields(vendorName, r.sku, htmlToText(page.html));
+    if (!ex || !ex.fields || !Object.keys(ex.fields).length) { continue; }
+    extracted++;
+    // build an UPDATE over only real, price/spec columns the model returned
+    const set = [], vals = [];
+    const map = { price: 'price', width_in: 'width_in', repeat_in: 'repeat_in',
+                  material: 'material', pattern_name: 'pattern_name', discontinued: 'discontinued' };
+    for (const [k, col] of Object.entries(map)) {
+      if (ex.fields[k] != null && cols.has(col)) { vals.push(ex.fields[k]); set.push(`${col} = $${vals.length}`); }
+    }
+    if (!set.length) continue;
+    if (sample.length < 5) sample.push({ sku: r.sku, via: ex.provider, fields: ex.fields });
+    if (!dryRun) {
+      if (cols.has('updated_at')) set.push(`updated_at = now()`);
+      vals.push(r.sku);
+      await pool.query(`UPDATE ${table} SET ${set.join(', ')} WHERE ${skuCol} = $${vals.length}`, vals);
+      updated++;
+    }
+    await new Promise(s => setTimeout(s, 1200 + Math.floor((Date.now() % 800)))); // politeness
+  }
+  return { table, vendorName, rows: rows.length, fetched, extracted, updated, failed, dryRun, sample };
+}
+
+module.exports = { refreshVendor, fetchHtml, htmlToText, extractFields };
+
+// CLI: node lib/generic-url-refresh.js <table> <vendorName> [limit] [--run]
+//   default = DRY-RUN (writes nothing). --run performs live UPDATEs to local staging (no publish).
+if (require.main === module) {
+  (async () => {
+    const argv = process.argv.slice(2);
+    const flags = argv.filter(a => a.startsWith('--'));
+    const pos = argv.filter(a => !a.startsWith('--'));
+    const [table = 'thibaut_catalog', vendorName = 'Thibaut', limit = '3'] = pos;
+    const dryRun = !flags.includes('--run');
+    const backend = await enrich.resolveProvider();
+    console.log(`  local backend: ${backend} | table=${table} | ${dryRun ? 'DRY-RUN (writes nothing)' : 'LIVE (updates local staging, no publish)'} | $0 local`);
+    const r = await refreshVendor({ table, vendorName, limit: parseInt(limit, 10), dryRun });
+    console.log(JSON.stringify(r, null, 2));
+    await pool.end().catch(() => {});
+  })();
+}
diff --git a/lib/resolve-pipeline.js b/lib/resolve-pipeline.js
index 82749f0..1ae7c68 100644
--- a/lib/resolve-pipeline.js
+++ b/lib/resolve-pipeline.js
@@ -26,7 +26,10 @@ function candidateNames(v) {
 // Returns { kind, cmd, cwd, reason }.
 //   kind = 'refresh-script'  -> runnable bash run.sh
 //   kind = 'unresolved'      -> no pipeline; caller SKIPS + logs
+const DISPATCHER_ROOT = path.join(__dirname, '..');
+
 function resolve(v) {
+  // 1. Hand-wired <code>-refresh/run.sh takes PRIORITY (Thibaut-style, for hard vendors).
   for (const name of candidateNames(v)) {
     const dir = path.join(cfg.REFRESH_SCRIPTS_DIR, `${name}-refresh`);
     const runSh = path.join(dir, 'run.sh');
@@ -36,13 +39,23 @@ function resolve(v) {
       }
     } catch (_) { /* keep probing */ }
   }
+  // 2. GENERIC product_url refresher (DTD Option C) — covers any vendor with a catalog_table.
+  //    Fetch each existing SKU's product_url (free) -> extract via LOCAL models ($0) -> update
+  //    staging. The generic refresher self-skips if the table has no product_url column/rows.
+  if (v.catalog_table && /^[a-z0-9_]+$/i.test(v.catalog_table)) {
+    const limit = cfg.GENERIC_LIMIT || 25;
+    return {
+      kind: 'refresh-script',
+      cmd: ['node', 'lib/generic-url-refresh.js', v.catalog_table, v.vendor_name, String(limit), '--run'],
+      cwd: DISPATCHER_ROOT,
+      reason: `generic product_url refresh (${v.catalog_table})`,
+    };
+  }
   return {
     kind: 'unresolved',
     cmd: null,
     cwd: null,
-    reason: `no scripts/<code>-refresh/run.sh for vendor_code=${v.vendor_code}` +
-            (v.pm2_name ? ` (pm2=${v.pm2_name})` : '') +
-            ' — wire this vendor\'s *-scraper-manager into a refresh dir',
+    reason: `no <code>-refresh/run.sh and no catalog_table for vendor_code=${v.vendor_code}`,
   };
 }
 
diff --git a/tools/fix-slack-webhook.js b/tools/fix-slack-webhook.js
new file mode 100644
index 0000000..05ad153
--- /dev/null
+++ b/tools/fix-slack-webhook.js
@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+// Fleet fix for the '${SLACK_WEBHOOK_URL}' literal bug: 95 DW scripts have
+//   const SLACK_WEBHOOK = '${SLACK_WEBHOOK_URL}';   // shell placeholder never expanded in JS
+// so their send functions crash on `new URL(SLACK_WEBHOOK)` AFTER doing their work.
+//
+// This applies TWO precise, safe transforms per file:
+//   1. assignment -> read the real env var:  process.env.SLACK_WEBHOOK_URL || ''
+//   2. guard the send: insert `if (!/^https?:\/\//.test(SLACK_WEBHOOK)) return;`
+//      immediately before the FIRST `new URL(SLACK_WEBHOOK)` (always inside the async
+//      send function, so an early return is valid) — so an unset webhook = clean no-op.
+// Idempotent: skips files already carrying the guard marker.
+const fs = require('fs');
+
+const files = process.argv.slice(2);
+const LIT = "const SLACK_WEBHOOK = '${SLACK_WEBHOOK_URL}';";
+const FIXED_ASSIGN = "const SLACK_WEBHOOK = process.env.SLACK_WEBHOOK_URL || '';";
+const GUARD = "if (!/^https?:\\/\\//.test(SLACK_WEBHOOK)) return; // slack: no-op when unset";
+
+let changed = 0, skipped = 0;
+for (const f of files) {
+  let s;
+  try { s = fs.readFileSync(f, 'utf8'); } catch { continue; }
+  if (s.includes('slack: no-op when unset')) { skipped++; continue; } // already fixed
+  if (!s.includes(LIT)) { skipped++; continue; }
+
+  let out = s.replace(LIT, FIXED_ASSIGN);
+  // guard the first `new URL(SLACK_WEBHOOK)` occurrence (with or without an assignment prefix)
+  out = out.replace(/(^[ \t]*)((?:const |let |var )?[\w$]*\s*=?\s*)?new URL\(SLACK_WEBHOOK\)/m,
+    (m, indent) => `${indent}${GUARD}\n${m}`);
+  if (out !== s) { fs.writeFileSync(f, out); changed++; console.log(`  fixed ${f.split('/').slice(-2).join('/')}`); }
+  else skipped++;
+}
+console.log(`\n  changed=${changed} skipped=${skipped}`);
← bec8dba add README + launchd plist draft + pending-approval go-live · back to Vendor Recrawl Dispatcher · (newest)