← back to Dw Unbuyable Recovery Pilot
TK-11041: the portal prices live per-colorway, not on the pattern page — add opt-in --deep
ecd384900c8320fefe2301314a9a36b8ad090df3 · 2026-09-11 11:26:37 -0700 · Steve Abrams
Found by chasing the discount forensics' own lead: DW already ran an AUTHENTICATED
Innovations crawl on 2026-06-17/18 (181 rows in innovations_catalog, price_source
'innovationsusa authed crawl 2026-06-18; list x0.90; per-yd'). Its crawler
(hollywood-import/innov-price.mjs) fetched /item/<pattern>/<sku> — per-COLORWAY URLs.
My script fetched only the 7 pattern pages, so a clean run could have returned
7x NOT_MEASURED and read as 'the vendor has no prices'.
--deep is opt-in, not a silent fallback, because robots Disallows /item/*/*. It prints
the conflict, throttles harder, and only fires for items whose pattern page had no price.
Default stays robots-clean.
Also settles the net-vs-list question with a hard artifact rather than the label heuristic:
the June crawl stored price_retail=134.95 (portal LIST) and price_trade=121.46 (list x 0.90),
so the portal shows LIST and the 10% applies on top.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CGo6pAQho6w9xgqghMgNKh
Files touched
M tk11041-innovations-reconcile/rescrape/innovations-trade-pull.js
Diff
commit ecd384900c8320fefe2301314a9a36b8ad090df3
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Fri Sep 11 11:26:37 2026 -0700
TK-11041: the portal prices live per-colorway, not on the pattern page — add opt-in --deep
Found by chasing the discount forensics' own lead: DW already ran an AUTHENTICATED
Innovations crawl on 2026-06-17/18 (181 rows in innovations_catalog, price_source
'innovationsusa authed crawl 2026-06-18; list x0.90; per-yd'). Its crawler
(hollywood-import/innov-price.mjs) fetched /item/<pattern>/<sku> — per-COLORWAY URLs.
My script fetched only the 7 pattern pages, so a clean run could have returned
7x NOT_MEASURED and read as 'the vendor has no prices'.
--deep is opt-in, not a silent fallback, because robots Disallows /item/*/*. It prints
the conflict, throttles harder, and only fires for items whose pattern page had no price.
Default stays robots-clean.
Also settles the net-vs-list question with a hard artifact rather than the label heuristic:
the June crawl stored price_retail=134.95 (portal LIST) and price_trade=121.46 (list x 0.90),
so the portal shows LIST and the 10% applies on top.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CGo6pAQho6w9xgqghMgNKh
---
.../rescrape/innovations-trade-pull.js | 50 +++++++++++++++++++++-
1 file changed, 48 insertions(+), 2 deletions(-)
diff --git a/tk11041-innovations-reconcile/rescrape/innovations-trade-pull.js b/tk11041-innovations-reconcile/rescrape/innovations-trade-pull.js
index 13b7c1a..f3f2191 100644
--- a/tk11041-innovations-reconcile/rescrape/innovations-trade-pull.js
+++ b/tk11041-innovations-reconcile/rescrape/innovations-trade-pull.js
@@ -15,13 +15,28 @@
// 10%, every DW sample request says "Net Less 20%"); a 10-point error propagates through
// cost/0.65/0.85 straight to customers. Settle TK-11041's discount question first.
// 4. No DB write. No Shopify write. Output is a staged JSON file for human review.
-// 5. robots.txt: /item/<slug>/ is Allow, /item/*/* is Disallow (and 500s). Pattern pages only.
+// 5. robots.txt: /item/<slug>/ is Allow, /item/*/* is Disallow (and 500s publicly). Default is
+// pattern pages ONLY. Prices, however, were historically found at the per-colorway URL (see
+// --deep below) — so a clean default run can legitimately come back NOT_MEASURED. That is a
+// real answer, not a failure, and it is the trigger to decide about --deep, not to guess.
//
// USAGE:
// node innovations-trade-pull.js --selftest # $0, no network, no session. Proves the
// # extractor REFUSES a logged-out page. Run first.
// node innovations-trade-pull.js --probe # login + auth proof + ONE pattern page (~1 session)
// node innovations-trade-pull.js # full pull, all 7 live pattern pages
+// node innovations-trade-pull.js --deep # ALSO fetch per-colorway /item/<slug>/<sku> for any
+// # item whose pattern page carried no price.
+//
+// ON --deep AND robots.txt — read this before using it. robots Disallows /item/*/*, so --deep is
+// OFF by default and must be turned on knowingly. The case for it: DW's own June 2026 authenticated
+// crawl priced 181 rows from exactly those per-colorway URLs (dw_unified.innovations_catalog,
+// price_source 'innovationsusa authed crawl 2026-06-18'), the pages are our own account's pricing
+// which the vendor explicitly told us to pull ("pricing is available through our website using your
+// login information"), and robots is a crawler-courtesy directive rather than an access control.
+// The case against: it is still a Disallow we would be stepping over. That is a judgment call for a
+// human, which is why it is a flag and not a fallback — and why --deep prints the conflict and
+// throttles harder. Default behaviour stays robots-clean.
const fs = require('fs');
const path = require('path');
@@ -158,6 +173,7 @@ function creds() {
const args = process.argv.slice(2);
if (args.includes('--selftest')) return selftest();
const probe = args.includes('--probe');
+ const deep = args.includes('--deep');
const { user, pass, url } = creds();
const map = JSON.parse(fs.readFileSync(IDENTITY_MAP, 'utf8'));
@@ -167,6 +183,15 @@ function creds() {
console.log(`[*] ${TICKET}: ${live.length} live items across ${pages.length} pattern pages` +
`${probe ? ' — PROBE: fetching 1' : ''}`);
console.log('[*] The 12 discontinued WHL/Whistler items are deliberately OUT of this pull (they 404).');
+ if (deep) {
+ console.log('[!] --deep ON: will also fetch per-colorway /item/<slug>/<sku> for items whose');
+ console.log(' pattern page carried no price. robots.txt Disallows /item/*/* — you have');
+ console.log(' chosen to step over that. Throttling harder. DW precedent: the 2026-06-18');
+ console.log(' authenticated crawl priced 181 rows from these same URLs.');
+ } else {
+ console.log('[*] robots-clean mode: pattern pages only. If prices turn out to live per-colorway,');
+ console.log(' expect NOT_MEASURED — that is the honest answer, and the cue to consider --deep.');
+ }
const Browserbase = require('@browserbasehq/sdk').default;
const { chromium } = require('playwright-core');
@@ -221,6 +246,27 @@ function creds() {
break;
}
}
+
+ // --- optional per-colorway pass, for items whose pattern page carried no price
+ if (deep && !aborted) {
+ const priced = new Set(results.filter(r => r.status === 'MEASURED').map(r => r.url));
+ const todo = live.filter(i => targets.some(([u]) => u === i.product_url) && !priced.has(i.product_url));
+ console.log(`\n[*] --deep: ${todo.length} item(s) still unpriced — trying per-colorway URLs`);
+ for (const item of todo) {
+ const slug = item.product_url.replace(/.*\/item\//, '');
+ const url_ = `https://www.innovationsusa.com/item/${slug}/${item.mfr_sku.toLowerCase()}`;
+ await page.goto(url_, { waitUntil: 'domcontentloaded' }).catch(() => {});
+ await page.waitForTimeout(THROTTLE_MS + 2000); // harder throttle on a Disallowed path
+ const text = await page.evaluate(() => document.body.innerText).catch(() => '');
+ const v = classifyPage(text);
+ console.log(` ${v.status.padEnd(15)} ${item.mfr_sku.padEnd(10)} ${v.price ? '$' + v.price.value + ' basis=' + v.price.basis : ''}`);
+ results.push({ mfr_sku: item.mfr_sku, pattern: item.innovations_pattern, url: url_, deep: true, ...v, chars: text.length });
+ if (v.status === 'SESSION_FAILURE') {
+ aborted = `session dropped at ${item.mfr_sku} during --deep — aborting rather than record false absences`;
+ break;
+ }
+ }
+ }
} catch (e) {
aborted = e.message;
console.error(`[!] ${e.message}`);
@@ -230,7 +276,7 @@ function creds() {
const ts = new Date().toISOString().replace(/[:.]/g, '').slice(0, 15) + 'Z';
const out = {
- ticket: TICKET, ts, mode: probe ? 'probe' : 'full',
+ ticket: TICKET, ts, mode: (probe ? 'probe' : 'full') + (deep ? '+deep' : ''),
session_id: session.id,
pages_attempted: targets.length, pages_returned: results.length,
measured: results.filter(r => r.status === 'MEASURED').length,
← ba3e4f9 TK-11041: one-line discount+basis question, ready to merge i
·
back to Dw Unbuyable Recovery Pilot
·
TK-11041: make the auth-proof failure self-diagnosing instea 0e76f2d →