`scripts/normalize.js`

150 lines
#!/usr/bin/env node
// Normalize raw Tokiwa scrape → data/products.json
// - mfr_sku cleaned from image alt (real Tokiwa code, INTERNAL ONLY)
// - deterministic pattern grouping (series) per category
// - deterministic Fentucci Naturals rename: pattern series → Italian coastal town
//   (data/rename-map.json is the committed stable mapping; existing entries are never reassigned)
// - DW SKUs DWFN-100001+ assigned in (category, mfr_sku) sort order
const fs = require('fs');
const path = require('path');

const ROOT = path.join(__dirname, '..');
const RAW = path.join(ROOT, 'data', 'raw');

const MATERIAL_BY_CATEGORY = {
  'raffia': 'Raffia', 'sisal': 'Sisal', 'abaca': 'Abaca', 'bamboo': 'Bamboo',
  'hyacinth': 'Water Hyacinth', 'fine-arrowroot': 'Fine Arrowroot', 'heavy-jute': 'Heavy Jute',
  'washi': 'Washi Paper', 'washi2': 'Washi Paper', 'japanese-silk': 'Japanese Silk',
  'japanese-silk2': 'Japanese Silk', 'metal-leaf': 'Metal Leaf', 'metalleaf2': 'Metal Leaf',
  'metal-leaf-3': 'Metal Leaf', 'mica': 'Mica', 'glassbeads': 'Glass Beads',
  'paperweave': 'Paperweave', 'denim': 'Denim', 'textile': 'Textile',
  'wood-veneer': 'Wood Veneer', 'gildingwashi': 'Gilding Washi',
};

// Italian coastal cities/towns — assignment order is fixed; never reshuffle.
const TOWNS = [
  'Portofino','Amalfi','Positano','Sorrento','Rimini','Vernazza','Manarola','Riomaggiore',
  'Corniglia','Monterosso','Camogli','Rapallo','Chiavari','Portovenere','Lerici','Levanto',
  'Alassio','Sanremo','Bordighera','Cervo','Noli','Varigotti','Savona','Zoagli','Recco',
  'Nervi','Bogliasco','Sori','Ravello','Praiano','Maiori','Minori','Cetara','Atrani',
  'Vietri','Salerno','Capri','Ischia','Procida','Taormina','Cefalu','Siracusa','Tropea',
  'Scilla','Gallipoli','Otranto','Polignano','Monopoli','Trani','Vieste','Ancona','Pesaro',
  'Senigallia','Numana','Sirolo','Grado','Caorle','Ravenna','Cesenatico','Cattolica',
  'Riccione','Sperlonga','Gaeta','Terracina','Anzio','Nettuno','Ladispoli','Orbetello',
  'Talamone','Viareggio','Sestri','Loano','Imperia','Ventimiglia','Celle','Albisola',
  'Arenzano','Bonassola','Moneglia','Deiva','Framura','Tellaro','Fiascherino','Ameglia',
];

// Towns that word-boundary-collide with pattern names in the EXISTING live/draft
// Fentucci family on the DW store (748 titles, checked 2026-07-22 by
// scripts/retown-collisions.py). nextTown() must never hand these out — they'd
// duplicate a live pattern name. Re-derive with retown-collisions.py --dry-run.
const LIVE_COLLIDING_TOWNS = new Set([
  'Portofino','Amalfi','Positano','Sorrento','Rimini','Camogli','Lerici',
  'Sanremo','Bordighera','Noli','Savona','Ravello','Atrani','Salerno','Capri',
  'Ischia','Procida','Taormina','Siracusa','Tropea','Scilla','Gallipoli',
  'Otranto','Ancona','Pesaro','Ravenna','Gaeta',
  // candidate-pool towns retown-collisions.py also found colliding:
  'Cervia','Chioggia','Fano','Ostuni','Anzio',
]);

// junk/hero alts that are not products (no mfr code exists — skip, never "Unknown")
const SKIP_ALTS = new Set([
  '11717431_890431071037161_121594246500198', // gildingwashi facebook hero
  '風景1', 'hito', 'stones02', 'Bark and Woodchip', 'WS312',
]);

function cleanSku(alt) {
  let s = alt.trim().replace(/\.(jpe?g|png|webp)$/i, '').trim();
  s = s.replace(/(_ps|_item|_up|_image|_s|-up)$/i, '');
  // photo-variant / secondary-code suffixes: RR02-01_F18 → RR02-01
  const us = s.indexOf('_');
  if (us > 2 && /^[A-Z]+\d*-\d+$/i.test(s.slice(0, us))) s = s.slice(0, us);
  return s.trim();
}

// deterministic pattern-series extraction
function seriesOf(sku) {
  let m;
  if ((m = sku.match(/^(.*\d)([A-D])$/))) return m[1];                       // PD-1949A → PD-1949
  if ((m = sku.match(/^(005-\d+-003)-\d+$/))) return m[1];                   // gildingwashi 005-801-003-310
  if ((m = sku.match(/^([A-Z -]+NO)\.\d+$/i))) return m[1].replace(/ NO$/i, ''); // J-SILK NO.5 → J-SILK
  if ((m = sku.match(/^([A-Za-z ]+)#\d+$/))) return m[1].trim();             // Silver Line#3 → Silver Line
  if ((m = sku.match(/^([A-Z]+\d+)-\d+$/i))) return m[1];                    // RR02-01 → RR02
  if ((m = sku.match(/^([A-Z]+(?:-[A-Z]+)?)-(\d{5})$/i))) return `${m[1]}-${m[2].slice(0, 3)}`; // A-18001 → A-180
  if ((m = sku.match(/^([A-Z]+(?:-[A-Z]+)?)-(\d{4})$/i))) return `${m[1]}-${m[2].slice(0, 2)}`; // GB-1037 → GB-10
  if ((m = sku.match(/^([A-Z]{1,4})(\d{4})$/i))) return m[1] + m[2].slice(0, 2); // KP3120 → KP31
  if ((m = sku.match(/^(\d{4})$/))) return sku.slice(0, 2);                  // 5601 → 56
  if ((m = sku.match(/^([A-Za-z]+(?:-[A-Za-z]+)?)[-# ]?\d+$/))) return m[1]; // FL-001 → FL, FE-PW001 → FE-PW
  return sku; // singleton pattern
}

// load or seed the stable rename map
const mapPath = path.join(ROOT, 'data', 'rename-map.json');
const renameMap = fs.existsSync(mapPath) ? JSON.parse(fs.readFileSync(mapPath)) : {};

const items = [];
const flags = [];
for (const f of fs.readdirSync(RAW).sort()) {
  if (f.startsWith('_')) continue;
  const d = JSON.parse(fs.readFileSync(path.join(RAW, f)));
  if (d.kind !== 'category') continue;
  for (const it of d.items) {
    const sku = cleanSku(it.alt);
    if (SKIP_ALTS.has(sku)) { flags.push({ category: d.category, alt: it.alt, reason: 'hero/non-product skipped' }); continue; }
    if (!sku || sku.length > 24) { flags.push({ category: d.category, alt: it.alt, reason: 'unusable alt skipped' }); continue; }
    items.push({
      mfr_sku: sku,
      source_category: d.category,
      source_page: d.source_page,
      image_url: it.image_url, // original full-res (transform params already stripped)
      source_alt: it.alt,
    });
  }
}

// dedupe mfr_sku across categories (same SKU can appear on two pages, e.g. GB codes) — first (sorted) category wins
items.sort((a, b) => a.source_category.localeCompare(b.source_category) || a.mfr_sku.localeCompare(b.mfr_sku));
const bySku = new Map();
for (const it of items) {
  if (bySku.has(it.mfr_sku)) flags.push({ category: it.source_category, alt: it.source_alt, reason: `duplicate mfr_sku (kept in ${bySku.get(it.mfr_sku).source_category})` });
  else bySku.set(it.mfr_sku, it);
}
const products = [...bySku.values()];

// pattern grouping + town assignment (stable: existing map entries never change)
const patternKeys = [...new Set(products.map(p => `${p.source_category}|${seriesOf(p.mfr_sku)}`))].sort();
const usedTowns = new Set(Object.values(renameMap).map(v => v.town));
let ti = 0;
function nextTown() {
  while (ti < TOWNS.length && (usedTowns.has(TOWNS[ti]) || LIVE_COLLIDING_TOWNS.has(TOWNS[ti]))) ti++;
  if (ti < TOWNS.length) return TOWNS[ti++];
  // overflow: Roman-suffixed second pass
  const n = Object.keys(renameMap).length;
  return `${TOWNS[n % TOWNS.length]} ${['II','III','IV'][Math.floor(n / TOWNS.length) - 1] || 'II'}`;
}
for (const key of patternKeys) {
  if (!renameMap[key]) { const t = nextTown(); renameMap[key] = { town: t }; usedTowns.add(t); }
}
fs.writeFileSync(mapPath, JSON.stringify(renameMap, null, 2));

// DW SKUs: DWFN-100001+ in (category, mfr_sku) order — already sorted above
let seq = 100001;
for (const p of products) {
  const key = `${p.source_category}|${seriesOf(p.mfr_sku)}`;
  p.pattern_series = seriesOf(p.mfr_sku);
  p.pattern_key = key;
  p.pattern_name = renameMap[key].town;
  p.material = MATERIAL_BY_CATEGORY[p.source_category];
  p.brand = 'Fentucci Naturals';
  p.dw_sku = `DWFN-${seq++}`;
  p.image_local = `images/${p.mfr_sku.replace(/[^A-Za-z0-9.-]+/g, '_')}.jpg`;
}

fs.writeFileSync(path.join(ROOT, 'data', 'products.json'), JSON.stringify(products, null, 2));
fs.writeFileSync(path.join(ROOT, 'data', 'normalize-flags.json'), JSON.stringify(flags, null, 2));
console.log(`products: ${products.length}, patterns: ${patternKeys.length}, flags: ${flags.length}`);
const perCat = {};
for (const p of products) perCat[p.source_category] = (perCat[p.source_category] || 0) + 1;
console.log(perCat);