← back to Dw Unbuyable Recovery Pilot

tk11041-innovations-reconcile/rescrape/innovations-trade-pull.js

343 lines

// Innovations USA trade-portal price pull — TK-11041.
// Cracked/staged 2026-09-11. Runs ONLY when INNOVATIONS_* credentials exist.
//
// WHY: Hektor Martinez (Innovations) answered our 2026-09-02 price-list request with
// "Pricing is available through our website using your login information. Account # 58315."
// Public pages carry ZERO price tokens (verified across all 371 pattern pages), so the
// logged-in portal is the only path to net price for the 39 live-pattern PR-Innovations items.
//
// HARD RULES BAKED IN (these are the point of the script, not decoration):
//   1. A page with no price is NEVER a $0 price. It is either NOT_MEASURED (auth proven on
//      that page) or SESSION_FAILURE (auth not proven) — and SESSION_FAILURE aborts the run.
//   2. Login must be POSITIVELY proven before any pattern page is fetched. Absence of an
//      error is not proof of a session.
//   3. This script does NOT compute retail. The cost basis is disputed (vendor_registry says
//      10%, every DW sample request says "Net Less 20%"); a 10-point error propagates through
//      cost/0.65/0.85 straight to customers. Settle TK-11041's discount question first.
//   4. No DB write. No Shopify write. Output is a staged JSON file for human review.
//   5. robots.txt: /item/<slug>/ is Allow, /item/*/* is Disallow (and 500s publicly). Default is
//      pattern pages ONLY. Prices, however, were historically found at the per-colorway URL (see
//      --deep below) — so a clean default run can legitimately come back NOT_MEASURED. That is a
//      real answer, not a failure, and it is the trigger to decide about --deep, not to guess.
//
// USAGE:
//   node innovations-trade-pull.js --selftest   # $0, no network, no session. Proves the
//                                               # extractor REFUSES a logged-out page. Run first.
//   node innovations-trade-pull.js --probe      # login + auth proof + ONE pattern page (~1 session)
//   node innovations-trade-pull.js              # full pull, all 7 live pattern pages
//   node innovations-trade-pull.js --deep       # ALSO fetch per-colorway /item/<slug>/<sku> for any
//                                               # item whose pattern page carried no price.
//
// ON --deep AND robots.txt — read this before using it. robots Disallows /item/*/*, so --deep is
// OFF by default and must be turned on knowingly. The case for it: DW's own June 2026 authenticated
// crawl priced 181 rows from exactly those per-colorway URLs (dw_unified.innovations_catalog,
// price_source 'innovationsusa authed crawl 2026-06-18'), the pages are our own account's pricing
// which the vendor explicitly told us to pull ("pricing is available through our website using your
// login information"), and robots is a crawler-courtesy directive rather than an access control.
// The case against: it is still a Disallow we would be stepping over. That is a judgment call for a
// human, which is why it is a flag and not a fallback — and why --deep prints the conflict and
// throttles harder. Default behaviour stays robots-clean.
const fs = require('fs');
const path = require('path');

const TICKET = 'TK-11041';
const REPO = process.env.INN_REPO ||
  path.join(process.env.HOME, 'Projects/dw-unbuyable-recovery-pilot/tk11041-innovations-reconcile/rescrape');
const IDENTITY_MAP = path.join(REPO, 'data/innovations-identity-map.json');
const OUT_DIR = path.join(REPO, 'data');
const FIXTURE_LOGGED_OUT = path.join(REPO, 'fixtures/logged-out-costine.html');

const LOGIN_URL_DEFAULT = 'https://www.innovationsusa.com/login';
const THROTTLE_MS = 350; // matches the public sweep's courtesy rate

// ---------------------------------------------------------------- extraction
// Authentication is asserted on EVERY page, not just at login: a portal can drop a session
// mid-run and keep serving 200s. "SIGN IN" present with no account marker == logged out.
// HONEST LIMITATION: nobody here has seen this site logged in. The June 2026 crawl saved only
// cookies (hollywood-import/innov-auth.json) — no localStorage, no captured markup — so these
// markers are an INFORMED GUESS at what the authenticated chrome says, not a verified fact.
// If they are wrong, every page would report SESSION_FAILURE and the run would look exactly
// like bad credentials. That ambiguity is the danger, so on the FIRST auth-proof failure the
// script DUMPS what the page actually says (see authDiagnostic) instead of just failing — one
// run tells you the real marker. Override without editing code:
//   INNOVATIONS_AUTH_POSITIVE='sign out|logout|account #' node innovations-trade-pull.js --probe
const AUTH_POSITIVE = new RegExp(
  process.env.INNOVATIONS_AUTH_POSITIVE || 'sign\\s*out|log\\s*out|logout|my\\s*account|account\\s*#|58315', 'i');
const AUTH_NEGATIVE = new RegExp(
  process.env.INNOVATIONS_AUTH_NEGATIVE || 'sign\\s*in|log\\s*in', 'i');

// What the page says about identity, so a wrong guess above is a 60-second fix rather than a
// dead end that reads as "the credentials must be wrong".
function authDiagnostic(text) {
  const lines = String(text).split(/\n+/).map(l => l.trim()).filter(Boolean);
  const hits = lines.filter(l => l.length < 80 &&
    /sign|log|account|welcome|hello|my |dealer|trade|cart|58315|\bhi\b/i.test(l)).slice(0, 14);
  return {
    note: 'Auth markers are a GUESS (nobody has seen this site logged in). If the lines below ' +
          'show you ARE logged in, set INNOVATIONS_AUTH_POSITIVE to a phrase from them and re-run.',
    first_lines: lines.slice(0, 8),
    identity_like_lines: hits,
    matched_positive: AUTH_POSITIVE.source,
    matched_negative: AUTH_NEGATIVE.source,
  };
}

function authState(text) {
  const pos = AUTH_POSITIVE.test(text);
  const neg = AUTH_NEGATIVE.test(text);
  if (pos && !neg) return 'AUTHENTICATED';
  if (pos && neg) return 'AMBIGUOUS';   // treated as failure — never trusted
  return 'LOGGED_OUT';
}

// Is this OUR NET price or the LIST price? Getting this wrong is the same class of error as the
// 10%-vs-20% discount conflict: the number is real, it is just a number about a DIFFERENT thing.
// A logged-in trade portal may show either. We never guess — UNKNOWN is a legitimate, reportable
// answer and forces a human to look once, rather than a silent assumption repricing 39 products.
const NET_HINT  = /\b(your\s*(net|price)|net\s*price|trade\s*price|dealer|wholesale|net\s*less)\b/i;
const LIST_HINT = /\b(list\s*price|msrp|retail\s*price|suggested)\b/i;

function priceBasis(context) {
  const net = NET_HINT.test(context), list = LIST_HINT.test(context);
  if (net && !list) return 'NET';
  if (list && !net) return 'LIST';
  return 'UNKNOWN'; // both or neither — do not pick one
}

// Returns {value, unit, raw, basis, context} or null. NEVER returns 0, never invents a unit,
// never assumes a basis.
function extractPrice(text) {
  const re = /\$\s?([0-9][0-9,]*(?:\.[0-9]{1,2})?)\s*(?:\/|per\s+)?\s*(yard|yd|lineal\s*yard|roll|sq\s*ft|each)?/i;
  const m = text.match(re);
  if (!m) return null;
  const value = parseFloat(m[1].replace(/,/g, ''));
  if (!Number.isFinite(value) || value <= 0) return null; // a $0 token is a defect, not a price
  const at = m.index || 0;
  const context = text.slice(Math.max(0, at - 120), at + 60).replace(/\s+/g, ' ').trim();
  return {
    value,
    unit: m[2] ? m[2].toLowerCase().replace(/\s+/g, ' ') : null,
    raw: m[0].trim(),
    basis: priceBasis(context),
    context, // kept verbatim so a human can adjudicate basis in one glance
  };
}

// One page -> one verdict. This is the whole safety contract in one function, so the
// self-test can exercise it with zero network.
function classifyPage(text) {
  const auth = authState(text);
  if (auth !== 'AUTHENTICATED') return { status: 'SESSION_FAILURE', auth, price: null };
  const price = extractPrice(text);
  if (!price) return { status: 'NOT_MEASURED', auth, price: null };
  return { status: 'MEASURED', auth, price };
}

// ---------------------------------------------------------------- self-test
// A positive-only test on a detector proves nothing. This injects the real logged-out page
// and demands the extractor go red. If this passes on a logged-out shell, the pull is unsafe.
function selftest() {
  let failures = 0;
  const check = (name, cond) => {
    console.log(`  ${cond ? 'PASS' : 'FAIL'}  ${name}`);
    if (!cond) failures++;
  };
  console.log(`[selftest] ${TICKET} innovations-trade-pull — negative tests`);

  if (fs.existsSync(FIXTURE_LOGGED_OUT)) {
    const html = fs.readFileSync(FIXTURE_LOGGED_OUT, 'utf8');
    const v = classifyPage(html);
    check('real logged-out /item/costine => SESSION_FAILURE', v.status === 'SESSION_FAILURE');
    check('real logged-out page yields NO price', v.price === null);
  } else {
    check(`fixture present at ${FIXTURE_LOGGED_OUT}`, false);
  }

  check('bare "Sign In" shell => SESSION_FAILURE',
    classifyPage('<a>Sign In</a> Costine').status === 'SESSION_FAILURE');
  check('authed page with NO price => NOT_MEASURED (never $0)',
    classifyPage('Sign Out | My Account — Costine CSO-001').status === 'NOT_MEASURED');
  check('a $0.00 token is rejected, not recorded as a price',
    classifyPage('Sign Out Account 58315 — Price $0.00').status === 'NOT_MEASURED');
  check('basis NET is read from the label, not assumed',
    classifyPage('Sign Out 58315 Your Net Price $84.50 per yard').price.basis === 'NET');
  check('basis LIST is read from the label, not assumed',
    classifyPage('Sign Out 58315 List Price $84.50 per yard').price.basis === 'LIST');
  check('an unlabelled price is UNKNOWN basis, never guessed NET',
    classifyPage('Sign Out 58315 Costine CSO-001 $84.50 per yard').price.basis === 'UNKNOWN');
  check('both NET and LIST labels present => UNKNOWN, not a coin flip',
    classifyPage('Sign Out 58315 List Price / Your Net Price $84.50').price.basis === 'UNKNOWN');
  check('authed page WITH price => MEASURED + numeric',
    (() => { const v = classifyPage('Sign Out Account 58315 — $84.50 per yard');
             return v.status === 'MEASURED' && v.price.value === 84.5 && /yard/.test(v.price.unit); })());
  check('BOTH sign-in and sign-out present => AMBIGUOUS, not trusted',
    classifyPage('Sign In Sign Out $99.00').status === 'SESSION_FAILURE');

  console.log(failures === 0 ? '[selftest] ALL PASS' : `[selftest] ${failures} FAILURE(S)`);
  process.exit(failures === 0 ? 0 : 1);
}

// ---------------------------------------------------------------- credentials
function creds() {
  require('dotenv').config({ path: path.join(__dirname, '.env') });
  const user = process.env.INNOVATIONS_USERNAME;
  const pass = process.env.INNOVATIONS_PASSWORD;
  const url = process.env.INNOVATIONS_LOGIN_URL || LOGIN_URL_DEFAULT;
  const missing = [!user && 'INNOVATIONS_USERNAME', !pass && 'INNOVATIONS_PASSWORD'].filter(Boolean);
  if (missing.length) {
    console.error(`[!] ${TICKET}: missing ${missing.join(', ')}.`);
    console.error('    These are registered in secrets-manager/routes.json and route here automatically.');
    console.error('    Steve pastes them once, the `secrets` skill fans them out. Do NOT hand-edit .env.');
    console.error('    Refusing to run: a credential-less run can only produce false absences.');
    process.exit(2);
  }
  return { user, pass, url };
}

// ---------------------------------------------------------------- main
(async () => {
  const args = process.argv.slice(2);
  if (args.includes('--selftest')) return selftest();
  const probe = args.includes('--probe');
  const deep = args.includes('--deep');

  const { user, pass, url } = creds();
  const map = JSON.parse(fs.readFileSync(IDENTITY_MAP, 'utf8'));
  const live = map.items.filter(i => i.site_state === 'live');
  const pages = [...new Map(live.map(i => [i.product_url, i.innovations_pattern])).entries()];
  const targets = probe ? pages.slice(0, 1) : pages;
  console.log(`[*] ${TICKET}: ${live.length} live items across ${pages.length} pattern pages` +
              `${probe ? ' — PROBE: fetching 1' : ''}`);
  console.log('[*] The 12 discontinued WHL/Whistler items are deliberately OUT of this pull (they 404).');
  if (deep) {
    console.log('[!] --deep ON: will also fetch per-colorway /item/<slug>/<sku> for items whose');
    console.log('    pattern page carried no price. robots.txt Disallows /item/*/* — you have');
    console.log('    chosen to step over that. Throttling harder. DW precedent: the 2026-06-18');
    console.log('    authenticated crawl priced 181 rows from these same URLs.');
  } else {
    console.log('[*] robots-clean mode: pattern pages only. If prices turn out to live per-colorway,');
    console.log('    expect NOT_MEASURED — that is the honest answer, and the cue to consider --deep.');
  }

  const Browserbase = require('@browserbasehq/sdk').default;
  const { chromium } = require('playwright-core');
  const bb = new Browserbase({ apiKey: process.env.BROWSERBASE_API_KEY });
  const session = await bb.sessions.create({
    projectId: process.env.BROWSERBASE_PROJECT_ID,
    browserSettings: { solveCaptchas: true }, // login page carries reCAPTCHA
  });
  console.log(`[*] browserbase session ${session.id}`);
  const browser = await chromium.connectOverCDP(session.connectUrl);
  const ctx = browser.contexts()[0];
  const page = ctx.pages()[0] || await ctx.newPage();
  page.setDefaultTimeout(45000);

  const results = [];
  let aborted = null;
  let diagnostic = null;
  try {
    // --- login
    await page.goto(url, { waitUntil: 'domcontentloaded' });
    await page.waitForTimeout(3000);
    const pw = await page.$('input[type=password]');
    if (pw) {
      const userSel = await page.$('input[type=email], input[name=email], input[name=username]');
      if (!userSel) throw new Error('login page has a password field but no resolvable username field');
      await userSel.fill(user);
      await pw.fill(pass);
      await page.waitForTimeout(400);
      await page.click('button[type=submit], input[type=submit], button:has-text("Sign In")')
        .catch(async () => { await pw.focus(); await page.keyboard.press('Enter'); });
      await page.waitForTimeout(9000); // captcha solve + redirect
    }

    // --- AUTH PROOF. Nothing is fetched until the session is positively proven.
    const homeText = await page.evaluate(() => document.body.innerText);
    const proof = authState(homeText);
    if (proof !== 'AUTHENTICATED') {
      diagnostic = authDiagnostic(homeText);
      console.error('\n[!] login NOT proven (auth=' + proof + '). Refusing to fetch pages — an ' +
                    'unproven session produces false absences that look exactly like "the vendor ' +
                    'has no price".');
      console.error('[?] BUT this may be MY marker guess being wrong rather than a bad login.');
      console.error('    What the page actually says:');
      for (const l of diagnostic.identity_like_lines) console.error('      | ' + l);
      console.error('    If those show you ARE logged in, re-run with:');
      console.error("      INNOVATIONS_AUTH_POSITIVE='<a phrase from above>' node innovations-trade-pull.js --probe");
      throw new Error(`login NOT proven (auth=${proof}) — see diagnostic above and in the staged JSON`);
    }
    console.log('[+] session AUTHENTICATED (positively proven, not merely error-free)');

    // --- pull
    for (const [url_, pattern] of targets) {
      await page.goto(url_, { waitUntil: 'domcontentloaded' });
      await page.waitForTimeout(THROTTLE_MS + 1200);
      const text = await page.evaluate(() => document.body.innerText);
      const v = classifyPage(text);
      console.log(`  ${v.status.padEnd(15)} ${pattern.padEnd(12)} ${v.price ? '$' + v.price.value + (v.price.unit ? '/' + v.price.unit : '') + '  basis=' + v.price.basis : ''}`);
      results.push({ pattern, url: url_, ...v, chars: text.length });
      if (v.status === 'SESSION_FAILURE') {
        aborted = `session dropped at ${pattern} — aborting rather than recording ${targets.length - results.length} false absences`;
        break;
      }
    }

    // --- optional per-colorway pass, for items whose pattern page carried no price
    if (deep && !aborted) {
      const priced = new Set(results.filter(r => r.status === 'MEASURED').map(r => r.url));
      const todo = live.filter(i => targets.some(([u]) => u === i.product_url) && !priced.has(i.product_url));
      console.log(`\n[*] --deep: ${todo.length} item(s) still unpriced — trying per-colorway URLs`);
      for (const item of todo) {
        const slug = item.product_url.replace(/.*\/item\//, '');
        const url_ = `https://www.innovationsusa.com/item/${slug}/${item.mfr_sku.toLowerCase()}`;
        await page.goto(url_, { waitUntil: 'domcontentloaded' }).catch(() => {});
        await page.waitForTimeout(THROTTLE_MS + 2000); // harder throttle on a Disallowed path
        const text = await page.evaluate(() => document.body.innerText).catch(() => '');
        const v = classifyPage(text);
        console.log(`  ${v.status.padEnd(15)} ${item.mfr_sku.padEnd(10)} ${v.price ? '$' + v.price.value + '  basis=' + v.price.basis : ''}`);
        results.push({ mfr_sku: item.mfr_sku, pattern: item.innovations_pattern, url: url_, deep: true, ...v, chars: text.length });
        if (v.status === 'SESSION_FAILURE') {
          aborted = `session dropped at ${item.mfr_sku} during --deep — aborting rather than record false absences`;
          break;
        }
      }
    }
  } catch (e) {
    aborted = e.message;
    console.error(`[!] ${e.message}`);
  } finally {
    await browser.close().catch(() => {});
  }

  const ts = new Date().toISOString().replace(/[:.]/g, '').slice(0, 15) + 'Z';
  const out = {
    ticket: TICKET, ts, mode: (probe ? 'probe' : 'full') + (deep ? '+deep' : ''),
    session_id: session.id,
    pages_attempted: targets.length, pages_returned: results.length,
    measured: results.filter(r => r.status === 'MEASURED').length,
    not_measured: results.filter(r => r.status === 'NOT_MEASURED').length,
    basis_unknown: results.filter(r => r.price && r.price.basis === 'UNKNOWN').length,
    aborted,
    auth_diagnostic: diagnostic,
    retail_computed: false,
    retail_blocked_reason:
      'TWO separate unknowns, both of which must be closed before retail is computed. ' +
      '(1) DISCOUNT: vendor_registry.innovations=10.00 vs "Net Less 20%" on every DW sample request ' +
      'for acct 58315. (2) BASIS: whether the portal number is our NET or the LIST price — see ' +
      'basis_unknown and each result.price.context. If the portal shows LIST, the discount applies ' +
      'on top; if NET, it does not. Applying the wrong one is a silent cost-basis error that ' +
      'propagates straight through cost/0.65/0.85 to customers.',
    target_columns: {
      note: 'innovations_catalog already has the columns; the write itself stays gated.',
      net_price: 'price_trade', unit: 'price_unit', retail: 'our_price',
      provenance: "price_source = 'innovations portal <date>'", stamped: 'price_updated_at',
    },
    results,
  };
  const outPath = path.join(OUT_DIR, `portal-pull-${ts}.json`);
  fs.writeFileSync(outPath, JSON.stringify(out, null, 2));
  console.log(`\n[*] staged -> ${outPath}`);
  console.log(`[*] measured ${out.measured}/${targets.length} pages · NO db write · NO shopify write · NO retail computed`);
  if (aborted) { console.error(`[!] RUN INCOMPLETE: ${aborted}`); process.exit(3); }
})();