PJ vendor_catalog spec scraper: tear-sheet specs for all phillip_jeffries rows (logo-placeholder publish rule, no images)

6b33ebc17f7573b08c169a194218c1a5389732a4 · 2026-06-10 08:42:38 -0700 · SteveStudio2

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Files touched

A DW-Programming/pj-vendor-spec-scraper.js

Diff

commit 6b33ebc17f7573b08c169a194218c1a5389732a4
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date:   Wed Jun 10 08:42:38 2026 -0700

    PJ vendor_catalog spec scraper: tear-sheet specs for all phillip_jeffries rows (logo-placeholder publish rule, no images)
    
    Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 DW-Programming/pj-vendor-spec-scraper.js | 205 +++++++++++++++++++++++++++++++
 1 file changed, 205 insertions(+)

diff --git a/DW-Programming/pj-vendor-spec-scraper.js b/DW-Programming/pj-vendor-spec-scraper.js
new file mode 100644
index 00000000..62b8ec52
--- /dev/null
+++ b/DW-Programming/pj-vendor-spec-scraper.js
@@ -0,0 +1,205 @@
+#!/usr/bin/env node
+/**
+ * Phillip Jeffries vendor_catalog Spec Scraper (2026-06-10)
+ *
+ * Goal: EVERY phillip_jeffries row in vendor_catalog has full tear-sheet specs.
+ * Images are NOT scraped — PJ goes online with the standing PJ logo (Steve rule).
+ *
+ * Targets rows where:
+ *   - specs IS NULL / '{}'  (never detail-scraped), OR
+ *   - specs lacks 'Fire Rating' AND last_scraped_at < RUN_CUTOFF (old 11-key scrape)
+ *
+ * Item-number resolution order:
+ *   1. pj_catalog.pj_item_number (join on mfr_sku)
+ *   2. mfr_sku itself when it looks like a PJ item (e.g. "071", "10000", "9837W")
+ *   3. trailing number in product_url
+ *
+ * Idempotent + resumable: every attempt stamps last_scraped_at, so re-runs skip
+ * everything already visited this run-day. Failures logged to pj-spec-scrape.log.
+ */
+
+const https = require('https');
+const fs = require('fs');
+const { Pool } = require('pg');
+
+const pool = process.env.DATABASE_URL
+  ? new Pool({ connectionString: process.env.DATABASE_URL })
+  : new Pool({ host: '/tmp', database: 'dw_unified' }); // local socket, peer auth
+
+const RUN_CUTOFF = '2026-06-10 00:00:00';
+const DELAY_MS = 2000;
+const LOG = `${__dirname}/pj-spec-scrape.log`;
+const PROGRESS = `${__dirname}/pj-spec-scrape-progress.json`;
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+const log = (msg) => {
+  const line = `${new Date().toISOString()} ${msg}`;
+  console.log(line);
+  fs.appendFileSync(LOG, line + '\n');
+};
+
+function getCardCode(itemNumber) {
+  return new Promise((resolve) => {
+    const req = https.request(
+      `https://www.phillipjeffries.com/${encodeURIComponent(itemNumber)}`,
+      { method: 'HEAD', timeout: 15000 },
+      (res) => {
+        if (res.statusCode === 301 || res.statusCode === 302) {
+          const m = (res.headers.location || '').match(/\/shop\/(CARD-[^/]+)\//);
+          resolve(m ? m[1] : null);
+        } else resolve(null);
+      }
+    );
+    req.on('error', () => resolve(null));
+    req.on('timeout', () => { req.destroy(); resolve(null); });
+    req.end();
+  });
+}
+
+function fetchTearSheet(cardCode, itemNumber) {
+  return new Promise((resolve) => {
+    https
+      .get(
+        `https://www.phillipjeffries.com/shop/${cardCode}/${encodeURIComponent(itemNumber)}/tear-sheet`,
+        { timeout: 20000 },
+        (res) => {
+          if (res.statusCode !== 200) { res.resume(); resolve(null); return; }
+          let data = '';
+          res.on('data', (c) => (data += c));
+          res.on('end', () => resolve(data));
+        }
+      )
+      .on('error', () => resolve(null))
+      .on('timeout', function () { this.destroy(); resolve(null); });
+  });
+}
+
+const decodeEntities = (s) =>
+  s.replace(/&#39;/g, "'").replace(/&amp;/g, '&').replace(/&quot;/g, '"').replace(/&lt;/g, '<').replace(/&gt;/g, '>');
+
+function parseSpecs(html) {
+  const specs = {};
+  const re = /<div class="detail--row">\s*<div class="detail--label[^"]*">([^<]+)<\/div>\s*<div class="detail--item[^"]*">\s*([\s\S]*?)\s*<\/div>/g;
+  let m;
+  while ((m = re.exec(html)) !== null) {
+    const label = m[1].trim();
+    const value = decodeEntities(m[2].replace(/<[^>]+>/g, '').trim());
+    if (value) specs[label] = value;
+  }
+  // pattern name from tear-sheet title if present
+  const t = html.match(/<div class="tear--title[^"]*">\s*([^<]+)/);
+  if (t) specs._title = decodeEntities(t[1].trim());
+  return specs;
+}
+
+function mapColumns(specs) {
+  const c = {};
+  if (specs['Width']) c.width = specs['Width'];
+  if (specs['Bolt Size']) c.length = specs['Bolt Size'];
+  if (specs['Repeat']) c.pattern_repeat = specs['Repeat'];
+  else if (specs['Vertical Repeat'] || specs['Horizontal Repeat']) {
+    const parts = [];
+    if (specs['Vertical Repeat']) parts.push('V: ' + specs['Vertical Repeat']);
+    if (specs['Horizontal Repeat']) parts.push('H: ' + specs['Horizontal Repeat']);
+    c.pattern_repeat = parts.join(', ');
+  }
+  if (specs['Fire Rating']) { c.fire_rating = specs['Fire Rating']; c.fire_rating_us = specs['Fire Rating']; }
+  if (specs['Maintenance']) c.maintenance = specs['Maintenance'];
+  if (specs['Seaming']) c.match_type = specs['Seaming'];
+  if (specs['Content']) c.composition = specs['Content'];
+  return c;
+}
+
+async function getQueue() {
+  const { rows } = await pool.query(
+    `SELECT v.id, v.mfr_sku, v.product_url, p.pj_item_number
+       FROM vendor_catalog v
+       LEFT JOIN pj_catalog p ON p.mfr_sku = v.mfr_sku
+      WHERE v.vendor_code = 'phillip_jeffries'
+        AND v.last_scraped_at < $1
+        AND (
+          v.specs IS NULL OR v.specs = '{}'::jsonb
+          OR (coalesce(v.fire_rating,'') = '' AND coalesce(v.fire_rating_us,'') = '')
+        )
+      ORDER BY (v.specs IS NULL OR v.specs = '{}'::jsonb) DESC, v.id`,
+    [RUN_CUTOFF]
+  );
+  return rows;
+}
+
+function resolveItem(row) {
+  if (row.pj_item_number) return String(row.pj_item_number).trim();
+  if (/^[0-9]{2,6}[A-Z]{0,3}$/i.test(row.mfr_sku.trim())) return row.mfr_sku.trim();
+  const m = (row.product_url || '').match(/phillipjeffries\.com\/(?:shop\/CARD-[^/]+\/)?([0-9]{2,6}[A-Z]{0,3})\b/i);
+  return m ? m[1] : null;
+}
+
+async function updateRow(id, cols, specs, cardCode, item) {
+  const sets = [];
+  const vals = [];
+  let i = 1;
+  for (const f of ['width', 'length', 'pattern_repeat', 'fire_rating', 'fire_rating_us', 'maintenance', 'match_type', 'composition']) {
+    if (cols[f]) {
+      sets.push(`${f} = CASE WHEN ${f} IS NULL OR ${f} = '' THEN $${i} ELSE ${f} END`);
+      vals.push(cols[f]);
+      i++;
+    }
+  }
+  const { _title, ...cleanSpecs } = specs;
+  sets.push(`specs = COALESCE(specs,'{}'::jsonb) || $${i}::jsonb`); vals.push(JSON.stringify(cleanSpecs)); i++;
+  sets.push(`product_url = CASE WHEN product_url IS NULL OR product_url = '' THEN $${i} ELSE product_url END`);
+  vals.push(`https://www.phillipjeffries.com/shop/${cardCode}/${item}`); i++;
+  sets.push(`last_scraped_at = now()`, `specs_changed_at = now()`, `updated_at = now()`);
+  vals.push(id);
+  await pool.query(`UPDATE vendor_catalog SET ${sets.join(', ')} WHERE id = $${i}`, vals);
+}
+
+async function stampVisited(id, reason) {
+  await pool.query(`UPDATE vendor_catalog SET last_scraped_at = now(), updated_at = now() WHERE id = $1`, [id]);
+  log(`FAIL id=${id}: ${reason}`);
+}
+
+async function main() {
+  const queue = await getQueue();
+  log(`=== PJ vendor spec scraper start: ${queue.length} rows in queue ===`);
+  let ok = 0, fail = 0, noItem = 0;
+  const t0 = Date.now();
+
+  for (let n = 0; n < queue.length; n++) {
+    const row = queue[n];
+    const item = resolveItem(row);
+    if (!item) { noItem++; await stampVisited(row.id, `no-item-number sku=${row.mfr_sku}`); continue; }
+
+    try {
+      const cardCode = await getCardCode(item);
+      if (!cardCode) { fail++; await stampVisited(row.id, `no-card-code item=${item}`); await sleep(DELAY_MS); continue; }
+      await sleep(500);
+      const html = await fetchTearSheet(cardCode, item);
+      if (!html) { fail++; await stampVisited(row.id, `no-tearsheet item=${item} card=${cardCode}`); await sleep(DELAY_MS); continue; }
+      const specs = parseSpecs(html);
+      if (!Object.keys(specs).filter((k) => k !== '_title').length) {
+        fail++; await stampVisited(row.id, `no-specs-parsed item=${item}`); await sleep(DELAY_MS); continue;
+      }
+      await updateRow(row.id, mapColumns(specs), specs, cardCode, item);
+      ok++;
+    } catch (e) {
+      fail++;
+      await stampVisited(row.id, `exception item=${item}: ${e.message}`);
+    }
+
+    if ((n + 1) % 25 === 0) {
+      const rate = (n + 1) / ((Date.now() - t0) / 60000);
+      const eta = Math.round((queue.length - n - 1) / rate);
+      fs.writeFileSync(PROGRESS, JSON.stringify({ done: n + 1, total: queue.length, ok, fail, noItem, etaMin: eta, at: new Date().toISOString() }));
+      log(`progress ${n + 1}/${queue.length} ok=${ok} fail=${fail} noItem=${noItem} eta=${eta}min`);
+    }
+    await sleep(DELAY_MS);
+  }
+
+  fs.writeFileSync(PROGRESS, JSON.stringify({ done: queue.length, total: queue.length, ok, fail, noItem, finished: true, at: new Date().toISOString() }));
+  log(`=== DONE ok=${ok} fail=${fail} noItem=${noItem} ===`);
+  await pool.end();
+  // exit code 0 = drained; wrapper decides whether anything retryable remains
+}
+
+main().catch((e) => { log('FATAL ' + e.stack); process.exit(1); });

← 06ea863b feat(dedar): feed-first collection extraction returns real p · back to Designer Wallcoverings · china_seas scraper: capture all_images=[imageUrl] (single-im 90279f43 →