`tools/pattern-grouping-dryrun.js`

118 lines
#!/usr/bin/env node
// WS-1 dry-run (READ-ONLY): compare candidate pattern-grouping strategies on the
// live dw_unified mirror so /dtd can commit a grouping-key choice on real numbers.
// Writes NOTHING. Outputs a report to stdout + tools/pattern-grouping-report.json.
//
//   node tools/pattern-grouping-dryrun.js
//
// Strategies compared (all scoped WITHIN vendor to avoid cross-vendor collisions):
//   T0  exact title
//   T1  normalized title (strip " | vendor" suffix, product-type noise, punctuation)
//   T2  T1 + strip color tokens (so "Albany Linen-Beige" and "-Cedar" collapse)
//   PN  pattern_name (baseline — known unreliable; quantifies how bad)
require('dotenv').config();
const fs = require('fs');
const path = require('path');
const { Pool } = require('pg');
const { COLORS } = require('../lib/classify');

const pool = new Pool({ connectionString: process.env.DATABASE_URL });

// product-type / material noise to drop from titles when deriving a pattern key
const NOISE = [
  'wallcovering','wallcoverings','wallpaper','wall covering','fabric','fabrics',
  'durable vinyl','vinyl','grasscloth','print','commercial','residential',
  'type ii','type iii','mural','wall mural','peel and stick','peel-and-stick',
  'self adhesive','self-adhesive','by phillipe romano','phillipe romano'
];
// color tokens = the classify COLOR set + common modifiers
const COLOR_WORDS = new Set([...COLORS,
  'light','dark','deep','pale','soft','warm','cool','muted','bright','antique',
  'metallic','natural','neutral','multi','multicolor','multicolour'
]);

function stripVendorSuffix(title) {
  // "Name ... | Vendor"  → "Name ..."
  const i = title.indexOf('|');
  return (i === -1 ? title : title.slice(0, i)).trim();
}
function normTitle(title) {
  let s = stripVendorSuffix(title).toLowerCase();
  for (const n of NOISE) s = s.split(n).join(' ');
  s = s.replace(/[^a-z0-9 ]+/g, ' ').replace(/\s+/g, ' ').trim();
  return s;
}
function stripColors(normalized) {
  const kept = normalized.split(' ').filter(w => w && !COLOR_WORDS.has(w));
  return kept.join(' ').trim() || normalized; // never empty-out a title
}

function summarize(label, keyFn, rows) {
  const groups = new Map(); // key -> {count, vendor, sampleTitles:Set}
  let keyed = 0;
  for (const r of rows) {
    const k = keyFn(r);
    if (!k) continue;
    keyed++;
    const gk = (r.vendor || '∅') + '::' + k;
    let g = groups.get(gk);
    if (!g) { g = { count: 0, vendor: r.vendor, key: k, samples: [] }; groups.set(gk, g); }
    g.count++;
    if (g.samples.length < 3) g.samples.push(r.title);
  }
  const arr = [...groups.values()];
  const multi = arr.filter(g => g.count >= 2);
  const productsInMulti = multi.reduce((a, g) => a + g.count, 0);
  arr.sort((a, b) => b.count - a.count);
  return {
    label,
    products_keyed: keyed,
    distinct_groups: arr.length,
    multi_colorway_groups: multi.length,
    products_in_multi_groups: productsInMulti,
    pct_collapsed: ((productsInMulti - multi.length) / keyed * 100).toFixed(1) + '%',
    avg_group_size: (keyed / arr.length).toFixed(2),
    top20: arr.slice(0, 20).map(g => ({ n: g.count, vendor: g.vendor, key: g.key, eg: g.samples })),
    // over-grouping suspects = very large groups (could be legit like Fine Stripes×76 OR junk)
    huge_groups_gt60: arr.filter(g => g.count > 60).length
  };
}

(async () => {
  console.log('Loading active products from mirror…');
  const r = await pool.query(`
    SELECT dw_sku, title, vendor, pattern_name
    FROM shopify_products
    WHERE status = 'ACTIVE' AND title IS NOT NULL AND handle IS NOT NULL
  `);
  const rows = r.rows;
  console.log('Active products:', rows.length);

  const reports = [
    summarize('T0 exact title',            x => stripVendorSuffix(x.title).toLowerCase(), rows),
    summarize('T1 normalized title',       x => normTitle(x.title), rows),
    summarize('T2 normalized − colors',    x => stripColors(normTitle(x.title)), rows),
    summarize('PN pattern_name (baseline)',x => (x.pattern_name || '').toLowerCase().trim() || null, rows)
  ];

  const out = { generated_at: new Date().toISOString(), active_products: rows.length, strategies: reports };
  const file = path.join(__dirname, 'pattern-grouping-report.json');
  fs.writeFileSync(file, JSON.stringify(out, null, 2));

  // ---- console digest ----
  for (const s of reports) {
    console.log('\n========================================');
    console.log(s.label);
    console.log('  distinct groups        :', s.distinct_groups);
    console.log('  multi-colorway groups  :', s.multi_colorway_groups);
    console.log('  products in those      :', s.products_in_multi_groups);
    console.log('  grid rows removed       :', (s.products_in_multi_groups - s.multi_colorway_groups), '(' + s.pct_collapsed + ' of catalog collapsed)');
    console.log('  avg group size         :', s.avg_group_size);
    console.log('  huge groups (>60)      :', s.huge_groups_gt60, '(inspect for over-grouping)');
    console.log('  top 5 largest:');
    s.top20.slice(0, 5).forEach(g => console.log(`    ${g.n}× [${g.vendor}] "${g.key}"  e.g. ${JSON.stringify(g.eg[0])}`));
  }
  console.log('\nFull report → ', file);
  await pool.end();
})().catch(e => { console.error(e); process.exit(1); });