← back to All Designerwallcoverings
crawler: read co-located Basic-Auth-gated microsite feeds via localhost internalViewer (bypass 401, no credential) + optional MICROSITE_BASIC_AUTH; Pierre Frey now shows 1161 on all.dw
97594827d54e3221006c728e6e59461d9b4f2391 · 2026-07-07 13:49:14 -0700 · steve@designerwallcoverings.com
Files touched
A scripts/adapters/contrado.jsA scripts/adapters/designtex.jsA scripts/adapters/fabricut.jsA scripts/adapters/koroseal.jsA scripts/adapters/schumacher.jsM scripts/crawl-microsites.js
Diff
commit 97594827d54e3221006c728e6e59461d9b4f2391
Author: steve@designerwallcoverings.com <steve@designerwallcoverings.com>
Date: Tue Jul 7 13:49:14 2026 -0700
crawler: read co-located Basic-Auth-gated microsite feeds via localhost internalViewer (bypass 401, no credential) + optional MICROSITE_BASIC_AUTH; Pierre Frey now shows 1161 on all.dw
---
scripts/adapters/contrado.js | 73 ++++++++++++++
scripts/adapters/designtex.js | 161 +++++++++++++++++++++++++++++++
scripts/adapters/fabricut.js | 104 ++++++++++++++++++++
scripts/adapters/koroseal.js | 211 +++++++++++++++++++++++++++++++++++++++++
scripts/adapters/schumacher.js | 110 +++++++++++++++++++++
scripts/crawl-microsites.js | 26 +++--
6 files changed, 679 insertions(+), 6 deletions(-)
diff --git a/scripts/adapters/contrado.js b/scripts/adapters/contrado.js
new file mode 100644
index 0000000..1ad5c4d
--- /dev/null
+++ b/scripts/adapters/contrado.js
@@ -0,0 +1,73 @@
+// adapters/contrado.js — Contrado live-stock adapter (PUBLIC-SAFE, SESSION-FREE — MADE-TO-ORDER).
+//
+// HONEST VERDICT (mined from the contrado-scraper-manager skill + the Import scrapers
+// lib/scrapers/contrado-new-products-scraper.ts / contrado-general-scraper.ts, 2026-07-07):
+//
+// Contrado (contrado.com) is a CUSTOM PRINT-ON-DEMAND platform — NOT a stocked storefront.
+// Every product is a blank that is manufactured to order at purchase time. There is NO Shopify/Woo
+// `.js` stock feed, no schema.org Offer `availability`, and no visible inventory/stock markup on the
+// anonymous product page: the server-rendered PDP carries only `<title>`, `og:image`, and a "From $X"
+// price. In other words there is NO live-stock signal to read — the item is *always* orderable
+// (made to order), which is categorically different from "in stock" (real on-hand inventory).
+//
+// Consequences for this adapter (deliberate, honest design):
+// • It does NOT open a metered Browserbase session — there is nothing stock-shaped to scrape, so a
+// paid session would only ever return "unknown". Cost is $0 (no session opened).
+// • It never fakes `in_stock:true`. It returns `in_stock:null` with `availability:'made to order'`,
+// so the grid honestly reflects "made to order — no live stock signal" instead of a green dot.
+// • No login is used or needed. Contrado trade creds DO exist in the dw-price-stock vault
+// (prefix CONTRADO), but the login gates checkout/pricing, not any stock signal — so they are not
+// read here. (Referencing them would still not surface inventory, because none is published.)
+//
+// PUBLIC-SAFE CONTRACT: availability only. `price` is ALWAYS null — Contrado's public "From $X" is a
+// custom-print DTC price, not our retail, so it is never surfaced (same rule as schema-public /
+// shopify-public / woo-public).
+//
+// HONEST-DIM RECOMMENDATION (for build-coverage.js — NOT changed by this file):
+// Contrado should be classified live_capable:false with reason "made to order — no stock signal".
+// It currently sits in build-coverage's SCHEMA_PUBLIC set but fails the resolvability gate (0%), so
+// it already falls through to a dim NO_STOCK tier. The precise, honest home is a SCHEMA_PARSE_PENDING
+// (or equivalent) entry, e.g.:
+// ['contrado', 'Contrado is a custom print-on-demand platform — every product is made to order;
+// the anonymous PDP publishes no stock feed / schema.org availability, so there is
+// no live-stock signal to read (sold made-to-order)']
+// That renders the vendor dim with an accurate reason instead of an empty/unknown live check.
+const { resolveCatalogUrl } = require('./resolve');
+
+async function run({ sku, pool, catalogTable, log }) {
+ const as_of = new Date().toISOString();
+
+ // Opportunistic resolve — attach the product_url when the SKU aligns with the catalog, purely for
+ // context. It NEVER gates the verdict: Contrado is made-to-order regardless of which SKU, and (per
+ // the 2026-07-07 audit) the grid's "Contrado" display SKUs are print-on-demand apparel/accessory
+ // blanks that don't intersect contrado_catalog's wallcovering slugs, so this usually won't resolve.
+ let product_url = null;
+ try {
+ const r = await resolveCatalogUrl(pool, catalogTable || 'contrado_catalog', sku);
+ if (r && r.product_url) product_url = r.product_url;
+ } catch { /* resolution is optional — the made-to-order verdict does not depend on it */ }
+
+ if (log) log('contrado made-to-order (no session opened, $0)', product_url || '(sku unresolved)');
+
+ // No Browserbase session is opened → session_opened:false, cost $0. `available:true` = the adapter
+ // produced a definitive PUBLIC-SAFE result (the made-to-order verdict), not that stock was found.
+ return {
+ available: true,
+ session_opened: false,
+ vendor: 'Contrado',
+ in_stock: null, // no inventory concept — never fake a stock state
+ availability: 'made to order', // print-on-demand: always orderable, manufactured at purchase
+ made_to_order: true,
+ lead_time: 'made to order (print on demand)',
+ price: null, // public "From $X" is custom-print DTC, NOT our retail → never emitted
+ raw_stock_label: 'made to order — no live stock signal',
+ product_url,
+ as_of,
+ source: 'contrado-live',
+ // Surfaced so the grid / coverage layer can dim honestly rather than showing an unknown check.
+ dim_recommended: true,
+ dim_reason: 'made to order — no stock signal',
+ };
+}
+
+module.exports = { run };
diff --git a/scripts/adapters/designtex.js b/scripts/adapters/designtex.js
new file mode 100644
index 0000000..0cb2647
--- /dev/null
+++ b/scripts/adapters/designtex.js
@@ -0,0 +1,161 @@
+// adapters/designtex.js — Designtex PUBLIC live-stock adapter (LOGIN-FREE, schema-public style).
+//
+// Designtex (designtex.com) is a Steelcase-owned commercial/contract textile + wallcovering house on
+// a Magento storefront. Prior fleet audit tagged it schema-public-eligible: its ANONYMOUS product
+// page emits schema.org Product/Offer availability + Magento stock markup, so live stock is readable
+// WITHOUT a trade login. The trade portal (customer/account/create) gates CHECKOUT/pricing, not the
+// public availability signal — so NO credential vault is needed for this adapter (unlike thibaut.js).
+//
+// This is functionally the same read as scripts/adapters/schema-public.js — Designtex is already in
+// build-coverage.js's SCHEMA_PUBLIC set and routes there today. The ONLY reason it dims is the
+// resolvability gate: designtex_catalog's product_url values don't align with the grid's display SKUs
+// (0% resolvable in the 2026-07-07 audit). This adapter fixes that IN THE ADAPTER by, when the direct
+// catalog join misses, LEARNING the Designtex URL template from a real catalog row and substituting
+// the grid SKU's mfr_sku — the SKU→product_url logic mined from the Designtex scraper (which keys
+// designtex_catalog by mfr_sku like "3833-301" and stores the Magento product_url per row).
+//
+// PUBLIC-SAFE: availability only. Designtex publishes a to-the-trade price, which is NOT our retail,
+// so `price` is ALWAYS null — never emitted (same contract as schema-public / shopify-public).
+const { resolveCatalogUrl, NAV_RE, releaseSession } = require('./resolve');
+
+const CATALOG = 'designtex_catalog';
+// Designtex PDP host — env-overridable so a template correction never needs a code change.
+const PDP_HOST = process.env.DESIGNTEX_PDP_HOST || 'https://www.designtex.com';
+
+// SKU→product_url resolver with a Designtex-specific fallback.
+// 1. Direct: resolve.js bridges the grid SKU → dw_sku/mfr_sku → designtex_catalog.product_url.
+// 2. Fallback (the 0%-resolvability fix): recover the grid row's mfr_sku from shopify_products,
+// LEARN the URL template from any real designtex_catalog row that carries both mfr_sku +
+// product_url, and substitute this SKU's mfr_sku into that template. This resolves SKUs whose
+// own catalog row has a stale/empty product_url, using the vendor's own URL shape — no guessing.
+async function resolveDesigntexUrl(pool, sku, log) {
+ const direct = await resolveCatalogUrl(pool, CATALOG, sku);
+ if (!direct.error && direct.product_url) return { url: direct.product_url, how: 'catalog' };
+
+ // Recover the display row's mfr_sku (the Designtex pattern-color code, e.g. "3833-301").
+ let mfr = null;
+ try {
+ const { rows } = await pool.query(
+ `SELECT mfr_sku FROM shopify_products
+ WHERE upper(coalesce(dw_sku,''))=upper($1) OR upper(coalesce(variant_sku,''))=upper($1)
+ OR upper(coalesce(sku,''))=upper($1) LIMIT 1`, [sku]);
+ if (rows[0] && rows[0].mfr_sku) mfr = String(rows[0].mfr_sku).trim();
+ } catch { /* shopify_products lookup optional */ }
+ if (!mfr) return { error: direct.error || 'no mfr_sku for this Designtex SKU' };
+
+ // Learn the URL template from a real catalog row (any row whose product_url embeds its mfr_sku).
+ try {
+ const { rows } = await pool.query(
+ `SELECT mfr_sku, product_url FROM "${CATALOG}"
+ WHERE product_url IS NOT NULL AND product_url <> '' AND mfr_sku IS NOT NULL AND mfr_sku <> ''
+ LIMIT 200`);
+ for (const r of rows) {
+ const code = String(r.mfr_sku).trim();
+ if (code && r.product_url.includes(code)) {
+ // Substitute this SKU's code into the learned template.
+ const url = r.product_url.split(code).join(mfr);
+ if (log) log('synthesized designtex url via learned template');
+ return { url, how: 'synthesized' };
+ }
+ }
+ } catch { /* catalog sample optional */ }
+
+ // Last resort — the canonical Designtex Magento product path (env-overridable host).
+ return { url: `${PDP_HOST}/products/${encodeURIComponent(mfr)}`, how: 'canonical-path' };
+}
+
+const readAvail = (s) => {
+ if (/OutOfStock|SoldOut|Discontinued|BackOrder/i.test(s)) return false;
+ if (/InStock|LimitedAvailability|PreOrder|OnlineOnly|InStoreOnly/i.test(s)) return true;
+ return null;
+};
+
+async function run({ sku, pool, newSession, log }) {
+ let sessionOpened = false, browser = null, session = null, bb = null;
+ try {
+ // 1. Resolve the grid SKU → its Designtex PDP (catalog join, else learned-template synthesis).
+ const r = await resolveDesigntexUrl(pool, sku, log);
+ if (r.error) return { available: false, reason: r.error };
+ const url = r.url;
+
+ // 2. Metered Browserbase session — NO login (public availability signal).
+ const sess = await newSession();
+ ({ browser, session, bb } = sess); const page = sess.page;
+ sessionOpened = true;
+ log('bb session', session && session.id, '→', url, `(${r.how})`);
+
+ await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 45000 });
+ await page.waitForTimeout(5000); // let Magento's client-rendered stock state settle
+
+ // 3. schema.org JSON-LD — offers.availability is authoritative when present.
+ const ldRaw = await page.$$eval('script[type="application/ld+json"]', (els) => els.map((e) => e.textContent).join('\n')).catch(() => '');
+ let in_stock = null, ldAvail = null;
+ if (ldRaw) {
+ const m = ldRaw.match(/"availability"\s*:\s*"([^"]+)"/i);
+ if (m) { ldAvail = m[1]; in_stock = readAvail(ldAvail); }
+ }
+
+ // 4. product-scoped DOM / Magento stock markup (link[itemprop=availability] href, .stock_status …).
+ let domTxt = '';
+ if (in_stock === null) {
+ domTxt = await page.$$eval(
+ '[itemprop=availability], link[itemprop=availability], meta[itemprop=availability], .product [class*=stock], .product-info [class*=stock], .stock_status, [class*=stock_status], [class*=stock-status], [class*=availab], .stock',
+ (els) => els.map((e) => (e.getAttribute('href') || e.getAttribute('content') || e.getAttribute('title') || e.textContent || '').trim()).filter(Boolean).join(' | ')
+ ).catch(() => '');
+ const t = String(domTxt).replace(NAV_RE, '');
+ const tok = readAvail(t);
+ if (tok !== null) in_stock = tok;
+ else if (/out\s*of\s*stock|sold\s*out|unavailable|no longer available/i.test(t)) in_stock = false;
+ else if (/in\s*stock|in-?stock|available/i.test(t)) in_stock = true;
+ }
+
+ // 5. structured whole-document availability vote (schema.org tokens / JSON availability keys).
+ let voteLabel = null;
+ if (in_stock === null) {
+ const vote = await page.evaluate(() => {
+ const html = document.documentElement.innerHTML || '';
+ const n = (re) => (html.match(re) || []).length;
+ const inTok = n(/schema\.org\/(?:InStock|LimitedAvailability|PreOrder)/gi)
+ + n(/[Aa]vailability\\?"?\s*:\s*\\?"?(?:InStock|LimitedAvailability|PreOrder)/g);
+ const outTok = n(/schema\.org\/(?:OutOfStock|SoldOut|Discontinued|BackOrder)/gi)
+ + n(/[Aa]vailability\\?"?\s*:\s*\\?"?(?:OutOfStock|SoldOut|Discontinued|BackOrder)/g);
+ return { inTok, outTok };
+ }).catch(() => ({ inTok: 0, outTok: 0 }));
+ if (vote.inTok || vote.outTok) {
+ if (vote.outTok > 0 && vote.inTok === 0) in_stock = false;
+ else if (vote.inTok > 0) in_stock = true;
+ voteLabel = `${in_stock ? 'in stock' : 'out of stock'} (${vote.inTok} in / ${vote.outTok} out signals)`;
+ }
+ }
+
+ // 6. add-to-cart / order-sample CTA state (last resort — enabled CTA ⇒ purchasable).
+ if (in_stock === null) {
+ const cta = await page.$$eval(
+ 'button[name=add], button[type=submit], [class*=add-to], [class*=addto], [class*=tocart], a[href*=cart]',
+ (els) => els.filter((e) => /add to (cart|basket|bag|sample)|add sample|order (a )?sample|request (a )?sample/i.test(
+ [e.textContent, e.getAttribute('aria-label'), e.getAttribute('title')].filter(Boolean).join(' ')))
+ .map((e) => (e.disabled || /disabled/i.test(e.className) ? 'disabled' : 'enabled')).join(',')
+ ).catch(() => '');
+ if (/enabled/.test(cta)) in_stock = true;
+ else if (cta && !/enabled/.test(cta)) in_stock = false;
+ }
+
+ const label = voteLabel || (in_stock === true ? 'in stock' : in_stock === false ? 'out of stock'
+ : (ldAvail || (domTxt ? domTxt.slice(0, 60) : 'unknown')));
+
+ return {
+ available: true, session_opened: true, vendor: 'Designtex',
+ in_stock, lead_time: null,
+ price: null, // to-the-trade price is NOT our retail → never emitted
+ raw_stock_label: label,
+ as_of: new Date().toISOString(),
+ source: 'designtex-live',
+ };
+ } catch (e) {
+ return { available: false, session_opened: sessionOpened, reason: 'live scrape error: ' + (e.message || e) };
+ } finally {
+ await releaseSession({ browser, bb, session });
+ }
+}
+
+module.exports = { run };
diff --git a/scripts/adapters/fabricut.js b/scripts/adapters/fabricut.js
new file mode 100644
index 0000000..0b362bb
--- /dev/null
+++ b/scripts/adapters/fabricut.js
@@ -0,0 +1,104 @@
+// adapters/fabricut.js — Fabricut trade-portal live-stock adapter (PORTAL + VAULT CREDS).
+//
+// Fabricut (fabricut.com) is a trade portal that also fronts Stroheim, S. Harris, Vervain, etc. under
+// fabricut.com/<brand>. The DW Fabricut scraper (fabricut-scraper-manager skill + the ImportNewSku
+// `fabricut-*` scrapers) scrapes the storefront ANONYMOUSLY via Browserbase/Puppeteer — it mines
+// pattern/color/sku/image from the PDP URL (`/<brand>/wallcovering/<sku>/<pattern>/<color>`) and does
+// NOT read any stock field; trade price is login-gated. The skill documents a trade login
+// (`https://fabricut.com/sign-in`) whose creds live ONLY in the dw-price-stock vault (prefix
+// FABRICUT). This adapter reuses that: it resolves creds by path via creds.js (never copied here),
+// logs in for ONE session, resolves the grid SKU → its Fabricut PDP via resolve.js, and reads
+// whatever availability the logged-in PDP exposes.
+//
+// PRICE-ORIENTED PORTAL CAVEAT: the mined scraper exposes NO live-stock field — Fabricut's value
+// behind the login is trade PRICE, not a reliable inventory count. So this adapter is best-effort on
+// availability (returns in_stock:null / label 'unknown' when the PDP shows no stock state) and, per
+// the PUBLIC-SAFE contract, ALWAYS emits price:null — the logged-in price is trade/net and never
+// leaves here.
+const { resolveCatalogUrl, releaseSession } = require('./resolve');
+const creds = require('./creds');
+
+const DEFAULT_LOGIN = 'https://fabricut.com/sign-in';
+
+async function run({ sku, pool, newSession, catalogTable, log }) {
+ let sessionOpened = false, browser = null, session = null, bb = null;
+ try {
+ // 1. Resolve the grid SKU → its Fabricut PDP (bridged via shopify_products → fabricut_catalog).
+ const r = await resolveCatalogUrl(pool, catalogTable || 'fabricut_catalog', sku);
+ if (r.error) return { available: false, reason: r.error };
+ const url = r.product_url;
+
+ // 2. Vault creds (prefix FABRICUT) — referenced in place; registry fallback. NEVER logged.
+ const c = await creds.resolve({ prefix: 'FABRICUT', pool, vendorLike: 'fabricut' });
+ if (!c || !c.pass) return { available: false, reason: 'fabricut trade credentials not available (vault)' };
+ const loginUrl = c.loginUrl || DEFAULT_LOGIN;
+
+ // 3. Metered Browserbase session + Fabricut trade login (email/username + password).
+ const sess = await newSession();
+ ({ browser, session, bb } = sess); const page = sess.page;
+ sessionOpened = true;
+ log('bb session', session && session.id);
+
+ await page.goto(loginUrl, { waitUntil: 'domcontentloaded', timeout: 45000 });
+ await page.waitForTimeout(2500);
+ // dismiss any cookie/consent gate that could overlay the form
+ for (const sel of ['button:has-text("Accept")', 'button:has-text("ACCEPT")', '#onetrust-accept-btn-handler']) {
+ const el = await page.$(sel).catch(() => null);
+ if (el) { await el.click().catch(() => {}); await page.waitForTimeout(600); break; }
+ }
+ const userEl = await page.$('#email, input[name=email], input[type=email], input[name=username], #username').catch(() => null);
+ if (userEl) {
+ await page.fill('#email, input[name=email], input[type=email], input[name=username], #username', c.user || '').catch(() => {});
+ await page.fill('#password, input[name=password], input[type=password]', c.pass).catch(() => {});
+ await page.waitForTimeout(400);
+ await page.click('button[type=submit], form button:has-text("Sign"), form button:has-text("Log")')
+ .catch(async () => { await page.press('#password, input[type=password]', 'Enter').catch(() => {}); });
+ // wait for the login page to fall away (redirect off /sign-in)
+ for (let i = 0; i < 18; i++) { await page.waitForTimeout(1000); if (!/sign-?in|log-?in|\/login/i.test(page.url())) break; }
+ }
+ const loggedIn = !/sign-?in|log-?in|\/login/i.test(page.url());
+
+ // 4. PDP → wait for the trade-gated content to hydrate, then read whatever availability shows.
+ await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 45000 });
+ await page.waitForTimeout(7000);
+
+ const html = await page.content().catch(() => '');
+ // Any embedded stock flags Fabricut might expose in inline JSON (best-effort; not observed in the
+ // anonymous scraper, so these commonly do not match — that's fine, they only ADD a signal).
+ const availForSale = /"availableForSale"\s*:\s*true/i.test(html) ? true
+ : /"availableForSale"\s*:\s*false/i.test(html) ? false : null;
+ const stockM = html.match(/"(?:stock|inventory|quantity)(?:_?on_?hand)?"\s*:\s*(\d+)/i);
+ const stockNum = stockM ? parseInt(stockM[1], 10) : null;
+ const backorder = /back\s*-?\s*order/i.test(html);
+
+ // product-scoped visible status text (avoid global nav chrome)
+ const availTxt = await page.$$eval(
+ '[itemprop=availability], main [class*=stock], main [class*=availab], main [class*=inventory], button:has-text("Add"), button:has-text("Sample")',
+ (els) => els.map((e) => (e.getAttribute('content') || e.getAttribute('aria-label') || e.textContent || '').trim()).filter(Boolean).join(' | ')
+ ).catch(() => '');
+
+ let in_stock = null;
+ if (availForSale === false || /out\s*of\s*stock|sold\s*out|discontinued|unavailable/i.test(availTxt)) in_stock = false;
+ else if (availForSale === true || stockNum > 0 || /in\s*stock|in-?stock|add to (cart|order|sample)/i.test(availTxt)) in_stock = true;
+ else if (stockNum === 0) in_stock = false;
+
+ const label = in_stock === true ? (backorder ? 'in stock (some backorder)' : 'in stock')
+ : in_stock === false ? 'out of stock'
+ : (availTxt ? availTxt.slice(0, 60) : 'unknown (Fabricut portal is price-oriented; no live stock field exposed)');
+
+ return {
+ available: true, session_opened: true, vendor: 'Fabricut',
+ in_stock, lead_time: backorder ? 'backorder possible' : null,
+ price: null, // Fabricut logged-in price is trade/net → never emitted
+ raw_stock_label: label,
+ as_of: new Date().toISOString(),
+ source: loggedIn ? 'fabricut-live' : 'fabricut-live (anon fallback)',
+ };
+ } catch (e) {
+ return { available: false, session_opened: sessionOpened, reason: 'live scrape error: ' + (e.message || e) };
+ } finally {
+ await releaseSession({ browser, bb, session });
+ }
+}
+
+module.exports = { run };
diff --git a/scripts/adapters/koroseal.js b/scripts/adapters/koroseal.js
new file mode 100644
index 0000000..09d83c9
--- /dev/null
+++ b/scripts/adapters/koroseal.js
@@ -0,0 +1,211 @@
+// adapters/koroseal.js — Koroseal live-stock/availability adapter (PUBLIC catalog + VAULT CREDS).
+//
+// WHAT KOROSEAL ACTUALLY IS (mined from the DW scraper engine —
+// Designer-Wallcoverings/DW-Programming/ImportNewSkufromURL/lib/scrapers/koroseal-spec-scraper.ts
+// + koroseal-general-scraper.ts, and colouranddesign-scraper-manager which "buys through Koroseal"):
+// koroseal.com is a COMMERCIAL/CONTRACT SPEC + SAMPLE catalog. The PDP exposes specs only —
+// Material, Backing, Fire Rating, Roll Width, Pattern Match, spec PDFs. There is NO cart, NO public
+// price, and NO stock/inventory field anywhere on the page (the DW koroseal microsite is flagged
+// `samples_only:true`). So Koroseal is NEITHER a live-price NOR a live-stock vendor in the Thibaut
+// sense. The only honest "live" signal the public site offers is PRODUCT LIVENESS: per the DW rule
+// "if a vendor product URL 301/404s or redirects to the category page → the product is
+// discontinued." So this adapter reports available (PDP resolves to a real colorway page) vs
+// discontinued (redirect-to-category / 404), and NEVER emits a price.
+//
+// CREDS: resolved by scripts/adapters/creds.js from the dw-price-stock vault (prefix KOROSEAL),
+// referenced IN PLACE — never copied here, never logged. koroseal.com scrapes anonymously, so login
+// is best-effort/defensive: attempted only if the vault supplies a KOROSEAL_LOGIN_URL. Availability
+// is read from the public PDP regardless of login.
+//
+// PUBLIC-SAFE: emits availability only. `price` is ALWAYS null — never emitted.
+const { resolveCatalogUrl, releaseSession } = require('./resolve');
+const creds = require('./creds');
+
+const BASE = 'https://koroseal.com';
+
+// ── SKU → product_url RECONSTRUCTION (the fix for Koroseal's 0% resolvability) ───────────────────
+// Ported from koroseal-spec-scraper.ts. Koroseal grid SKUs are mfr codes (e.g. "C921-10", "AL21-15",
+// "TAK-AA04-02", "B22-07"), but koroseal_catalog.product_url is a COLLECTION/COLOR PATH — the two do
+// not join on a bare SKU string, which is exactly why only 9/2351 rows resolve today. We rebuild the
+// URL from the SKU: prefix → collection slug, then match the exact colorway off the live collection
+// page's thumb strip (.thumbscontain a.thumbprev; span.tptext = SKU, href = the real product_url).
+
+// Regular wallcovering prefix → collection slug ( /products/wallcoverings/<collection>/<color> )
+const SKU_TO_COLLECTION = {
+ C921: 'chateau', C923: 'chateau', RA21: 'arco', SPR1: 'spire',
+ AL21: 'adalyn-weave', AS21: 'astra',
+ 'DK-WT': 'digital', 'DK-ST': 'digital', 'DK-': 'digital',
+ MB0: 'mandy-budan', 'SRD-': 'sarah-rowland',
+};
+
+// TAK Specialty prefix → collection ( /products/specialty/koroseal-specialty/<collection>/<sku> )
+const TAK_COLLECTIONS = {
+ 'TAK-AA01': 'subtle-leaf', 'TAK-AA02': 'first-impression', 'TAK-AA04': 'opulent-leaf',
+ 'TAK-AA05': 'steeped-leaf', 'TAK-AA08': 'languid-leaf', 'TAK-AA09': 'whispering-patina',
+ 'TAK-BA01': 'ruche', 'TAK-BB01': 'galaxy-wash', 'TAK-BB02': 'galaxy-wash-(1)',
+ 'TAK-CA01': 'color-wash-cork', 'TAK-CA02': 'molten-cork', 'TAK-CA03': 'molten-cork-stria',
+ 'TAK-CA04': 'aged-molten-cork', 'TAK-CA05': 'aura-cork', 'TAK-CA06': 'native-cork',
+ 'TAK-CA07': 'quercus', 'TAK-CB01': 'beveled-native-cork', 'TAK-DA01': 'afterglow',
+ 'TAK-EA01': 'cyrus-weave', 'TAK-EA02': 'kilika', 'TAK-EA03': 'silk-noil',
+ 'TAK-EC01': 'portmanteau', 'TAK-ED01': 'dapper-weave', 'TAK-EE01': 'beveled-raffia',
+ 'TAK-EE03': 'native-raffia', 'TAK-EE04': 'native-sisal', 'TAK-EF01': 'pulp-and-plaid',
+ 'TAK-EG01': 'whitecap-woven', 'TAK-EH01': 'floating-gossamer', 'TAK-FA01': 'bark-rind',
+ 'TAK-GA01': 'woven-wood',
+};
+
+const isTak = (s) => s.toUpperCase().startsWith('TAK-');
+
+function takCollection(sku) {
+ const u = sku.toUpperCase();
+ for (const [pfx, col] of Object.entries(TAK_COLLECTIONS)) if (u.startsWith(pfx)) return col;
+ return null;
+}
+function regularCollection(sku) {
+ const u = sku.toUpperCase();
+ for (const [pfx, col] of Object.entries(SKU_TO_COLLECTION)) if (u.startsWith(pfx)) return col;
+ return null;
+}
+
+// Candidate collection/search pages to mine a SKU's exact product_url from — most-specific first.
+function korosealCollectionPages(sku) {
+ const u = sku.toUpperCase();
+ const pages = [];
+ if (isTak(u)) {
+ const col = takCollection(u);
+ if (col) {
+ // TAK PDP path already includes the sku — try it directly, then the collection page.
+ pages.push(`${BASE}/products/specialty/koroseal-specialty/${col}/${u.toLowerCase()}`);
+ pages.push(`${BASE}/products/specialty/koroseal-specialty/${col}`);
+ }
+ } else {
+ const col = regularCollection(u);
+ if (col) pages.push(`${BASE}/products/wallcoverings/${col}`);
+ }
+ // Prefix search fallback (per DW koroseal rule): SG13-02 → search "SG13". Recovers collections we
+ // don't have a static prefix map for, so the reconstruction degrades gracefully instead of failing.
+ const pfx = (u.match(/^[A-Z]+[0-9]*/) || [u])[0];
+ pages.push(`${BASE}/search-results?searchtext=${encodeURIComponent(pfx)}`);
+ return pages;
+}
+
+// Read the collection/search page's thumb strip and return the href whose SKU === the grid SKU.
+async function matchThumbUrl(page, sku) {
+ const target = String(sku).toUpperCase().trim();
+ return page.$$eval('.thumbscontain a.thumbprev, a.thumbprev, a[href*="/products/"]', (els) =>
+ els.map((el) => {
+ const skuEl = el.querySelector('span.tptext');
+ const skuText = (skuEl && skuEl.childNodes[0] ? skuEl.childNodes[0].textContent : (skuEl ? skuEl.textContent : '')) || '';
+ return { sku: skuText.trim(), href: el.href || el.getAttribute('href') || '' };
+ }).filter((t) => t.sku && t.href)
+ ).then((thumbs) => {
+ const hit = thumbs.find((t) => t.sku.toUpperCase() === target)
+ || thumbs.find((t) => t.sku.toUpperCase().startsWith(target) || target.startsWith(t.sku.toUpperCase()));
+ return hit ? hit.href : null;
+ }).catch(() => null);
+}
+
+// Resolve the grid SKU → its koroseal.com PDP. Catalog join first (works once koroseal_catalog is
+// backfilled); on miss, live-reconstruct via the collection/thumb walk described above.
+async function resolveKorosealUrl(page, pool, catalogTable, sku) {
+ const r = await resolveCatalogUrl(pool, catalogTable || 'koroseal_catalog', sku).catch(() => ({ error: 'resolve failed' }));
+ if (r && r.product_url) return { url: r.product_url, via: 'catalog' };
+
+ // Fallback: reconstruct from the SKU by mining the live collection/search page's thumbs.
+ for (const cp of korosealCollectionPages(sku)) {
+ try {
+ await page.goto(cp, { waitUntil: 'domcontentloaded', timeout: 45000 });
+ await page.waitForTimeout(2500);
+ // If we navigated straight to a TAK PDP that exists, use it as-is.
+ if (/\/products\/specialty\/koroseal-specialty\/[^/]+\/[^/]+/.test(page.url()) && !/page-not-found|404/i.test(await page.content().catch(() => ''))) {
+ return { url: page.url(), via: 'reconstruct-direct' };
+ }
+ const href = await matchThumbUrl(page, sku);
+ if (href) return { url: href.startsWith('http') ? href : `${BASE}${href.startsWith('/') ? '' : '/'}${href}`, via: 'reconstruct-thumb' };
+ } catch { /* try next candidate */ }
+ }
+ return { error: (r && r.error) || 'SKU not resolvable to a Koroseal product_url' };
+}
+
+async function run({ sku, pool, newSession, catalogTable, log }) {
+ let sessionOpened = false, browser = null, session = null, bb = null;
+ try {
+ // 1. Vault creds (prefix KOROSEAL) — referenced in place; registry fallback. NEVER logged.
+ // Login is optional for Koroseal (public catalog); we resolve creds so a future gated portal
+ // still works, and so build-coverage's vaultHas('KOROSEAL') gate stays truthful.
+ const c = await creds.resolve({ prefix: 'KOROSEAL', pool, vendorLike: 'koroseal' });
+ const loginUrl = c && c.loginUrl ? c.loginUrl : null;
+
+ // 2. Metered Browserbase session.
+ const sess = await newSession();
+ ({ browser, session, bb } = sess); const page = sess.page;
+ sessionOpened = true;
+ log('bb session', session && session.id);
+
+ // 3. Best-effort trade login (only if the vault gave us a login URL + a password).
+ let loggedIn = false;
+ if (loginUrl && c && c.pass) {
+ try {
+ await page.goto(loginUrl, { waitUntil: 'domcontentloaded', timeout: 45000 });
+ await page.waitForTimeout(2000);
+ const emailEl = await page.$('#email, input[name=email], input[type=email], #username, input[name=username]').catch(() => null);
+ if (emailEl) {
+ await page.fill('#email, input[name=email], input[type=email], #username, input[name=username]', c.user || '').catch(() => {});
+ await page.fill('#password, input[name=password], input[type=password]', c.pass).catch(() => {});
+ await page.waitForTimeout(300);
+ await page.click('button[type=submit], form button:has-text("Sign"), form button:has-text("Log")')
+ .catch(async () => { await page.press('#password, input[type=password]', 'Enter').catch(() => {}); });
+ for (let i = 0; i < 12; i++) { await page.waitForTimeout(1000); if (!/login|sign-?in/i.test(page.url())) break; }
+ loggedIn = !/login|sign-?in/i.test(page.url());
+ }
+ } catch { /* public catalog works without login */ }
+ }
+
+ // 4. Resolve the grid SKU → its koroseal.com PDP (catalog join, else live reconstruction).
+ const res = await resolveKorosealUrl(page, pool, catalogTable, sku);
+ if (res.error) return { available: false, session_opened: true, reason: res.error };
+ const url = res.url;
+
+ // 5. Load the PDP and read AVAILABILITY (liveness). No price/stock field exists to read.
+ await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 45000 });
+ await page.waitForTimeout(4000);
+ const finalUrl = page.url();
+ const html = (await page.content().catch(() => '')) || '';
+ const bodyText = await page.evaluate(() => document.body ? document.body.innerText : '').catch(() => '');
+
+ // Discontinued signals: redirect off the colorway PDP to a bare category/list, or a 404/not-found.
+ const notFound = /page[-\s]?not[-\s]?found|404|no results|not found|we could(?:n'?t| not) find/i.test(html);
+ const redirectedToCategory =
+ /\/products\/wallcoverings\/?$/.test(finalUrl) ||
+ /\/products\/?$/.test(finalUrl) ||
+ /\/search-results/i.test(finalUrl) ||
+ /-list\/?$/.test(finalUrl);
+ // Live-product signals: the spec block Koroseal renders on a real colorway page.
+ const hasSpecs = /Material\s*:|Fire Rating\s*:|Roll Width|Pattern Match\s*:|Backing\s*:/i.test(bodyText)
+ || /\bthumbprev\b|\btppic\b|\btptext\b/.test(html);
+
+ let in_stock = null, availability = 'unknown (availability parse pending verification)';
+ if (notFound || redirectedToCategory) {
+ in_stock = false; availability = 'discontinued (PDP no longer resolves / redirected to category)';
+ } else if (hasSpecs) {
+ in_stock = true; availability = 'available (current in Koroseal catalog — spec/sample only, no live inventory count)';
+ }
+
+ return {
+ available: true, session_opened: true, vendor: 'Koroseal',
+ in_stock,
+ availability,
+ lead_time: null,
+ price: null, // Koroseal PDP carries NO public price — never emitted
+ raw_stock_label: availability,
+ as_of: new Date().toISOString(),
+ source: loggedIn ? 'koroseal-live' : 'koroseal-live',
+ resolved_via: res.via || null,
+ };
+ } catch (e) {
+ return { available: false, session_opened: sessionOpened, reason: 'live scrape error: ' + (e.message || e) };
+ } finally {
+ await releaseSession({ browser, bb, session });
+ }
+}
+
+module.exports = { run };
diff --git a/scripts/adapters/schumacher.js b/scripts/adapters/schumacher.js
new file mode 100644
index 0000000..7c7901e
--- /dev/null
+++ b/scripts/adapters/schumacher.js
@@ -0,0 +1,110 @@
+// adapters/schumacher.js — Schumacher (F. Schumacher) trade-portal live-stock adapter (PORTAL + VAULT CREDS).
+//
+// F. Schumacher (schumacher.com / fschumacher.com) is a login-gated trade line. The PUBLIC product
+// pages are quote-only — the DW Schumacher catalog carries no price and rolls read literally
+// "** Please Call for Pricing on F. Schumacher Products **". Anonymous PDPs therefore expose NO
+// live stock and NO retail; both live behind the trade login. Creds are resolved by
+// scripts/adapters/creds.js from the dw-price-stock vault (prefix SCHUMACHER) and referenced IN
+// PLACE — never copied here, never logged; they travel only in memory for this one login.
+//
+// STOCK EXPOSURE CAVEAT: the existing DWSW scraper (schumacher-dwsw-updater skill) only mines
+// pattern/color/spec/fire-rating/CDN-image data — it never reads availability, and Schumacher's
+// public pages are quote-only. Whether the trade PDP surfaces a machine-readable in-stock signal is
+// UNVERIFIED (no live proof session run). So this adapter reads availability DEFENSIVELY and
+// dims to in_stock:null ("unknown") rather than guessing when no stock signal is present.
+//
+// PUBLIC-SAFE: emits availability only. Schumacher's logged-in price is trade/net (and the line is
+// quote-only publicly), so `price` is ALWAYS null — cost/net/wholesale NEVER leave here.
+const { resolveCatalogUrl, releaseSession } = require('./resolve');
+const creds = require('./creds');
+
+// Prefer the vault's SCHUMACHER_LOGIN_URL (authoritative); this is only a last-resort default.
+const DEFAULT_LOGIN = 'https://www.fschumacher.com/account/login';
+
+async function run({ sku, pool, newSession, catalogTable, log }) {
+ let sessionOpened = false, browser = null, session = null, bb = null;
+ try {
+ // 1. Resolve the grid SKU → its Schumacher PDP (bridged via shopify_products → schumacher_catalog).
+ // Same 39%-resolvability path the coverage gate measures — SKUs with no catalog product_url
+ // simply return "not found" and the grid dims that row (the known SKU↔catalog alignment gap).
+ const r = await resolveCatalogUrl(pool, catalogTable || 'schumacher_catalog', sku);
+ if (r.error) return { available: false, reason: r.error };
+ const url = r.product_url;
+
+ // 2. Vault creds (prefix SCHUMACHER) — referenced in place; registry fallback. NEVER logged.
+ const c = await creds.resolve({ prefix: 'SCHUMACHER', pool, vendorLike: 'schumacher' });
+ if (!c || !c.pass) return { available: false, reason: 'schumacher trade credentials not available (vault)' };
+ const loginUrl = c.loginUrl || DEFAULT_LOGIN;
+
+ // 3. Metered Browserbase session + Schumacher trade login (email/username + password).
+ const sess = await newSession();
+ ({ browser, session, bb } = sess); const page = sess.page;
+ sessionOpened = true;
+ log('bb session', session && session.id);
+
+ await page.goto(loginUrl, { waitUntil: 'domcontentloaded', timeout: 45000 });
+ await page.waitForTimeout(2500);
+ // dismiss any cookie/consent gate that could overlay the form
+ for (const sel of ['button:has-text("Accept")', 'button:has-text("ACCEPT")', '#onetrust-accept-btn-handler']) {
+ const el = await page.$(sel).catch(() => null);
+ if (el) { await el.click().catch(() => {}); await page.waitForTimeout(600); break; }
+ }
+ const emailSel = '#email, input[name=email], input[type=email], #username, input[name=username]';
+ const passSel = '#password, input[name=password], input[type=password]';
+ const emailEl = await page.$(emailSel).catch(() => null);
+ if (emailEl) {
+ await page.fill(emailSel, c.user || '').catch(() => {});
+ await page.fill(passSel, c.pass).catch(() => {});
+ await page.waitForTimeout(400);
+ await page.click('button[type=submit], form button:has-text("Sign"), form button:has-text("Log"), form button:has-text("Continue")')
+ .catch(async () => { await page.press(passSel, 'Enter').catch(() => {}); });
+ // wait for the login page to fall away (redirect off /login | /account/login)
+ for (let i = 0; i < 18; i++) { await page.waitForTimeout(1000); if (!/\/(account\/)?login/i.test(page.url())) break; }
+ }
+ const loggedIn = !/\/(account\/)?login/i.test(page.url());
+
+ // 4. PDP → wait for any trade-gated inventory to hydrate, then read availability DEFENSIVELY.
+ await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 45000 });
+ await page.waitForTimeout(7000);
+
+ const html = await page.content().catch(() => '');
+ // Opportunistic embedded stock flags (present on many trade storefronts' product JSON).
+ const availForSale = /"availableForSale"\s*:\s*true/i.test(html) ? true
+ : /"availableForSale"\s*:\s*false/i.test(html) ? false : null;
+ const stockedM = html.match(/"(?:stocked|qtyAvailable|quantityAvailable|inventory_quantity)"\s*:\s*(\d+)/i);
+ const stocked = stockedM ? parseInt(stockedM[1], 10) : null;
+ const backorder = /"backorder"\s*:\s*(1|true)/i.test(html) || /back\s*-?order/i.test(html);
+
+ // product-scoped visible status text (avoid global nav "Check Stock"/"Discontinued" chrome)
+ const availTxt = await page.$$eval(
+ '[itemprop=availability], main [class*=stock], main [class*=availab], main [class*=inventory], button:has-text("Add"), button:has-text("Order")',
+ (els) => els.map((e) => (e.getAttribute('content') || e.getAttribute('aria-label') || e.textContent || '').trim()).filter(Boolean).join(' | ')
+ ).catch(() => '');
+
+ let in_stock = null;
+ if (availForSale === false || stocked === 0 || /out\s*of\s*stock|sold\s*out|discontinued|unavailable|no longer available/i.test(availTxt)) in_stock = false;
+ else if (availForSale === true || (typeof stocked === 'number' && stocked > 0) || /in\s*stock|in-?stock|add to (cart|order|sample|bag)/i.test(availTxt)) in_stock = true;
+ // else: no stock signal on the trade PDP → in_stock stays null ("unknown"), row dims gracefully.
+
+ const label = in_stock === true ? (backorder ? 'in stock (some backorder)' : 'in stock')
+ : in_stock === false ? 'out of stock'
+ : (availTxt ? availTxt.slice(0, 60) : 'unknown (no stock signal on trade PDP — verification pending)');
+
+ return {
+ available: true, session_opened: true, vendor: 'Schumacher',
+ in_stock,
+ availability: label, // task-requested alias
+ lead_time: backorder ? 'backorder possible' : null,
+ price: null, // Schumacher is quote-only / trade-net → NEVER emitted
+ raw_stock_label: label, // registry-contract field (matches thibaut shape)
+ as_of: new Date().toISOString(),
+ source: loggedIn ? 'schumacher-live' : 'schumacher-live (anon fallback)',
+ };
+ } catch (e) {
+ return { available: false, session_opened: sessionOpened, reason: 'live scrape error: ' + (e.message || e) };
+ } finally {
+ await releaseSession({ browser, bb, session });
+ }
+}
+
+module.exports = { run };
diff --git a/scripts/crawl-microsites.js b/scripts/crawl-microsites.js
index bc138dc..b38ddb7 100644
--- a/scripts/crawl-microsites.js
+++ b/scripts/crawl-microsites.js
@@ -42,7 +42,7 @@ const slugify = (v) => String(v || '').toLowerCase().normalize('NFD')
function seedFromRegistry() {
const reg = JSON.parse(fs.readFileSync(SEED, 'utf8'));
const out = [];
- for (const b of reg.brands || []) out.push({ slug: b.slug, vendor: b.vendor || null, type: 'brand', note: b.note || null });
+ for (const b of reg.brands || []) out.push({ slug: b.slug, vendor: b.vendor || null, type: 'brand', note: b.note || null, internalViewer: b.internalViewer || null });
for (const i of reg.internal || []) out.push({ slug: i.slug, vendor: null, type: 'internal', note: i.label || null });
return out;
}
@@ -97,8 +97,16 @@ function mergeSeeds(reg, db) {
}
// ── per-site fetch ───────────────────────────────────────────────────────────────
+// Many DW microsites are Basic-Auth gated (the 401 IS the healthy gate). Send the
+// shared internal cred (env MICROSITE_BASIC_AUTH="user:pass") so the crawler can read
+// their /api/products feed. External Shopify storefronts simply ignore the header.
+const MICROSITE_AUTH = process.env.MICROSITE_BASIC_AUTH
+ ? 'Basic ' + Buffer.from(process.env.MICROSITE_BASIC_AUTH).toString('base64')
+ : null;
async function getText(url, ms) {
- const r = await fetch(url, { redirect: 'follow', signal: AbortSignal.timeout(ms), headers: { 'user-agent': 'all-dw-crawler/1.0 (+all.designerwallcoverings.com)' } });
+ const headers = { 'user-agent': 'all-dw-crawler/1.0 (+all.designerwallcoverings.com)' };
+ if (MICROSITE_AUTH) headers.Authorization = MICROSITE_AUTH;
+ const r = await fetch(url, { redirect: 'follow', signal: AbortSignal.timeout(ms), headers });
return { status: r.status, ok: r.ok, finalUrl: r.url, body: await r.text() };
}
@@ -157,14 +165,17 @@ function buildFeedResult(arr, host, reportedTotal, truncated) {
return { has: true, count: reportedTotal != null ? reportedTotal : arr.length, truncated: !!truncated, handles, products };
}
-async function feedProducts(host) {
+async function feedProducts(host, feedBase) {
// Capture the FULL handle set per microsite (not a 24-sample) so membership is exact.
// • DW-family /api/products → returns the WHOLE catalog in one shot (ignores limit);
// shape { count|total, rows|products:[…] }.
// • Shopify /products.json → 250/page, paginate to completion.
+ // feedBase lets a co-located, Basic-Auth-gated microsite be read on localhost
+ // (http://127.0.0.1:<port>) — bypassing its public 401 with no credential.
+ const dwBase = feedBase || `https://${host}`;
// 1. DW-family family endpoint first.
try {
- const { status, body } = await getText(`https://${host}/api/products?limit=100000`, FULL_FEED_TIMEOUT_MS);
+ const { status, body } = await getText(`${dwBase}/api/products?limit=100000`, FULL_FEED_TIMEOUT_MS);
if (status === 200) {
const j = JSON.parse(body);
const arr = Array.isArray(j.rows) ? j.rows : Array.isArray(j.products) ? j.products : null;
@@ -192,6 +203,9 @@ async function feedProducts(host) {
async function crawlOne(seed) {
const host = `${seed.slug}.${BASE}`;
+ // Co-located Basic-Auth-gated microsite → read its feed on localhost, no credential.
+ const lv = (seed.internalViewer || '').match(/https?:\/\/(?:127\.0\.0\.1|localhost):\d+/);
+ const feedBase = lv ? lv[0] : null;
const rec = {
slug: seed.slug, host, url: `https://${host}/`, type: seed.type,
vendor: clean(seed.vendor), note: seed.note || null,
@@ -211,8 +225,8 @@ async function crawlOne(seed) {
if (root.body) { rec.title = metaTitle(root.body) || rec.title; rec.hero = ogImage(root.body) || rec.hero; }
} catch (e) { rec.error = e.name === 'TimeoutError' ? 'timeout' : e.message; }
- if (rec.up && rec.status !== 401) {
- const f = await feedProducts(host);
+ if (rec.up && (rec.status !== 401 || feedBase)) {
+ const f = await feedProducts(host, feedBase);
if (f.has) {
rec.feed = true;
rec.productCount = f.count;
← 3340f23 live-stock: enable Thibaut in coverage (vault-cred proof lan
·
back to All Designerwallcoverings
·
all.dw infinite-scroll: robust pump loop (keep loading while acfd50a →