`build.js`

169 lines
#!/usr/bin/env node
/*
 * TK-135 Crawl Atlas builder — joins the typed descriptor proposal files by gid
 * into a coverage summary + a color/style/motif/material co-occurrence graph.
 * $0, local, deterministic. No Shopify API calls.
 */
const fs = require('fs');
const path = require('path');

const SRC = path.join(process.env.HOME, 'Projects/Designer-Wallcoverings/shopify/scripts/data/tk135');
const OUT = path.join(__dirname, 'data/graph.json');

const readTSV = (f) => {
  const p = path.join(SRC, f);
  if (!fs.existsSync(p)) { console.error('MISSING', f); return []; }
  return fs.readFileSync(p, 'utf8').split('\n').filter(Boolean).map(l => l.split('\t'));
};
const gidNum = (g) => (g || '').replace('gid://shopify/Product/', '');
const stripColor = (c) => (c || '').replace(/^color:/i, '').trim();

// ---- gid universe: products that have a crawlable image -------------------
const universe = new Set(readTSV('active-images.tsv').map(r => gidNum(r[0])));

// ---- per-gid typed descriptors --------------------------------------------
const color = new Map();      // gid -> color name
const domHex = new Map();     // gid -> dominant hex (from palette)
const style = new Map();
const motif = new Map();
const material = new Map();

for (const [g, cRaw, palJson] of readTSV('palette-proposed.tsv')) {
  const gid = gidNum(g); if (!gid) continue;
  const c = stripColor(cRaw); if (c) color.set(gid, c);
  try { const pal = JSON.parse(palJson); if (pal && pal[0] && pal[0].hex) domHex.set(gid, pal[0].hex); } catch {}
}
for (const [g, s] of readTSV('style-proposed.tsv'))              { const gid = gidNum(g); if (gid && s) style.set(gid, s.trim()); }
for (const [g, m] of readTSV('motif-proposed-full.tsv'))         { const gid = gidNum(g); if (gid && m && !/^UNMATCHED/.test(m)) motif.set(gid, m.trim()); }
for (const [g, m] of readTSV('material-primary-proposed.tsv'))   { const gid = gidNum(g); if (gid && m) material.set(gid, m.trim()); }

// ---- TRUE live coverage: a descriptor also counts if the product's LIVE tags
//      already carry a value from that descriptor's vocabulary (vendor-provided).
//      Fill the per-gid maps from live tags too, so coverage + graph reflect the
//      full catalog, not just the title-match proposal subset.
const vocabOf = (map) => new Set([...map.values()]);
const Vs = vocabOf(style), Vm = vocabOf(motif), Vmat = vocabOf(material);
for (const [g, tagStr] of readTSV('active-tags.tsv')) {
  const gid = gidNum(g); if (!universe.has(gid)) continue;
  const tags = (tagStr || '').split(',').map(t => t.trim());
  if (!style.has(gid))    { const hit = tags.find(t => Vs.has(t));   if (hit) style.set(gid, hit); }
  if (!motif.has(gid))    { const hit = tags.find(t => Vm.has(t));   if (hit) motif.set(gid, hit); }
  if (!material.has(gid)) { const hit = tags.find(t => Vmat.has(t)); if (hit) material.set(gid, hit); }
}

// ---- coverage --------------------------------------------------------------
const total = universe.size;
let hasColor=0, hasStyle=0, hasMotif=0, hasMaterial=0, hasAll3=0;
for (const gid of universe) {
  const c = color.has(gid), s = style.has(gid), m = motif.has(gid), mat = material.has(gid);
  if (c) hasColor++; if (s) hasStyle++; if (m) hasMotif++; if (mat) hasMaterial++;
  if (c && s && m) hasAll3++;
}

// ---- co-occurrence ---------------------------------------------------------
// nodes keyed as "type:value"; edges = symmetric co-occurrence counts across gids
const CATS = { color, style, motif, material };
const nodeCount = {};       // key -> product count
const nodeHexAccum = {};     // color key -> [r,g,b,n] for averaging swatch
const edges = new Map();     // "keyA|keyB" (sorted) -> count

const hexToRgb = (h) => { const n = parseInt(h.slice(1),16); return [n>>16&255, n>>8&255, n&255]; };

for (const gid of universe) {
  const feats = [];
  for (const [type, map] of Object.entries(CATS)) {
    const v = map.get(gid);
    if (v) feats.push(`${type}:${v}`);
  }
  // count nodes + accumulate color swatch
  for (const k of feats) {
    nodeCount[k] = (nodeCount[k] || 0) + 1;
    if (k.startsWith('color:') && domHex.has(gid)) {
      const [r,g,b] = hexToRgb(domHex.get(gid));
      const a = nodeHexAccum[k] || [0,0,0,0];
      a[0]+=r; a[1]+=g; a[2]+=b; a[3]++; nodeHexAccum[k]=a;
    }
  }
  // edges between every distinct pair (skip same-type same-value trivially — different types only for cleaner map,
  // but also allow color-color? no, one value per type per gid). Pair across all feats.
  for (let i=0;i<feats.length;i++) for (let j=i+1;j<feats.length;j++) {
    const [a,b] = [feats[i],feats[j]].sort();
    const key = a+'|'+b;
    edges.set(key, (edges.get(key)||0)+1);
  }
}

// build adjacency: key -> {neighborKey: count}
const adj = {};
for (const [key, cnt] of edges) {
  const [a,b] = key.split('|');
  (adj[a] = adj[a] || {})[b] = cnt;
  (adj[b] = adj[b] || {})[a] = cnt;
}

// nodes list
const swatch = (k) => {
  const a = nodeHexAccum[k];
  if (!a || !a[3]) return null;
  const r=Math.round(a[0]/a[3]), g=Math.round(a[1]/a[3]), b=Math.round(a[2]/a[3]);
  return '#'+[r,g,b].map(x=>x.toString(16).padStart(2,'0')).join('');
};
const nodes = {};
for (const [k, n] of Object.entries(nodeCount)) {
  const [type, ...rest] = k.split(':');
  nodes[k] = { key:k, type, label: rest.join(':'), count:n };
  if (type==='color') nodes[k].hex = swatch(k) || '#888';
}

// ---- product images + a compact products index for the grid ---------------
const CDN = 'https://cdn.shopify.com';
const shortImg = (u) => (u || '').replace(CDN, '').replace(/\?.*$/, '');   // strip host + ?v= cache-buster
const imgOf = new Map();                          // gid -> short image path
for (const [g, url] of readTSV('active-images.tsv')) { const gid = gidNum(g); if (url) imgOf.set(gid, shortImg(url)); }
const titleOf = new Map();
for (const [g, title] of readTSV('active-full.tsv')) { const gid = gidNum(g); if (title) titleOf.set(gid, title); }

// products index: only imaged products, compact row per product
const prodRows = [];
for (const gid of universe) {
  const img = imgOf.get(gid); if (!img) continue;
  prodRows.push([gid, titleOf.get(gid) || '', img, color.get(gid) || '', style.get(gid) || '', motif.get(gid) || '', material.get(gid) || '']);
}
// give each node a representative sample image = first matching product's image
const sampleFor = {};
for (const [gid, , img, c, s, m, mat] of prodRows) {
  const put = (key) => { if (key && !sampleFor[key]) sampleFor[key] = img; };
  put('color:'+c); put('style:'+s); put('motif:'+m); put('material:'+mat);
}
for (const k of Object.keys(nodes)) if (sampleFor[k]) nodes[k].img = sampleFor[k];

const out = {
  builtAt: new Date().toISOString(),
  coverage: {
    total, hasColor, hasStyle, hasMotif, hasMaterial, hasAll3,
    pctColor:+(100*hasColor/total).toFixed(1),
    pctStyle:+(100*hasStyle/total).toFixed(1),
    pctMotif:+(100*hasMotif/total).toFixed(1),
    pctMaterial:+(100*hasMaterial/total).toFixed(1),
    pctAll3:+(100*hasAll3/total).toFixed(1),
  },
  counts: {
    colors: Object.values(nodes).filter(n=>n.type==='color').length,
    styles: Object.values(nodes).filter(n=>n.type==='style').length,
    motifs: Object.values(nodes).filter(n=>n.type==='motif').length,
    materials: Object.values(nodes).filter(n=>n.type==='material').length,
  },
  nodes,
  adj,
};
out.cdn = CDN;
fs.writeFileSync(OUT, JSON.stringify(out));
// products index as its own file (loaded once, filtered client-side for the grid)
const PROD = path.join(__dirname, 'data/products.json');
fs.writeFileSync(PROD, JSON.stringify({ cdn: CDN, cols:['g','t','img','color','style','motif','material'], rows: prodRows }));
console.log('coverage:', out.coverage);
console.log('counts:', out.counts);
console.log('nodes:', Object.keys(nodes).length, 'edges:', edges.size);
console.log('wrote', OUT, (fs.statSync(OUT).size/1024/1024).toFixed(2)+'MB');
console.log('wrote', PROD, (fs.statSync(PROD).size/1024/1024).toFixed(2)+'MB', '·', prodRows.length, 'products');