CLIP-1: /api/similar?engine=clip — image-embedding nearest-neighbour via visual-search service, graceful fallback to tag engine (opt-in, non-breaking)

52301eab591d5f4670ecb5facf1b67170b2ec647 · 2026-07-12 23:05:46 -0700 · Steve Abrams

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Files touched

M server.js

Diff

commit 52301eab591d5f4670ecb5facf1b67170b2ec647
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Sun Jul 12 23:05:46 2026 -0700

    CLIP-1: /api/similar?engine=clip — image-embedding nearest-neighbour via visual-search service, graceful fallback to tag engine (opt-in, non-breaking)
    
    Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 server.js | 100 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 89 insertions(+), 11 deletions(-)

diff --git a/server.js b/server.js
index 349af2f..ce8ce94 100644
--- a/server.js
+++ b/server.js
@@ -14,6 +14,12 @@ const DATABASE_URL = process.env.DATABASE_URL;
 const CORS_ORIGINS = (process.env.CORS_ORIGIN || '*')
   .split(',').map(s => s.trim()).filter(Boolean);
 
+// CLIP visual-search service (dw-photo-capture/visual-search/search_service.py).
+// Used by /api/similar?engine=clip for image-embedding nearest-neighbour. Optional:
+// if unreachable OR the source SKU has no stored vector, /api/similar transparently
+// falls back to the tag engine, so this is a pure enhancement.
+const VISUAL_SEARCH_URL = process.env.VISUAL_SEARCH_URL || 'http://127.0.0.1:9914';
+
 if (!DATABASE_URL) {
   console.error('FATAL: DATABASE_URL not set — copy .env.example to .env');
   process.exit(1);
@@ -241,6 +247,67 @@ function paletteOverlap(sourcePalette, candPalette) {
   return { score: total, pickedUp, avgDE };
 }
 
+// Batch-load enrichment rows for a set of dw_skus (for CLIP-result UI parity:
+// title, palette, vendor, handle). First row per dw_sku wins.
+async function loadRowsByDwSkus(skus) {
+  if (!skus || !skus.length) return [];
+  const q = `SELECT ${SOURCE_COLS}
+       FROM shopify_products sp
+       LEFT JOIN shopify_color_enrichment sce ON sce.shopify_id = sp.shopify_id
+       WHERE sp.dw_sku = ANY($1)`;
+  const r = await pool.query(q, [skus]);
+  const seen = new Set(), out = [];
+  for (const row of r.rows) {
+    if (!row.dw_sku || seen.has(row.dw_sku)) continue;
+    seen.add(row.dw_sku); out.push(row);
+  }
+  return out;
+}
+
+// CLIP nearest-neighbour "similar" via the visual-search service. Returns an array
+// in the SAME shape pickSimilar() produces (_score/_palette/_dE/_why) so the shared
+// palette+paint response tail is reused verbatim. Returns null on any miss (service
+// down, source SKU never embedded, empty result) → caller falls back to the tag engine.
+async function clipSimilarTop(source, k) {
+  if (!source.dw_sku) return null;
+  try {
+    const body = { dw_sku: source.dw_sku, hex: source.dominant_hex || '', k };
+    const r = await fetch(`${VISUAL_SEARCH_URL}/similar`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(4000)
+    });
+    if (!r.ok) return null;                              // 404 = no stored vector for this sku
+    const d = await r.json();
+    if (!d || !d.ok || !Array.isArray(d.results) || d.results.length === 0) return null;
+
+    const skus = d.results.map(x => x.dw_sku).filter(Boolean);
+    const rowByS = new Map((await loadRowsByDwSkus(skus)).map(row => [row.dw_sku, row]));
+    const sourcePalette = paletteOf(source);
+
+    return d.results.map(x => {
+      const row = rowByS.get(x.dw_sku) || null;
+      const candPalette = row ? paletteOf(row) : [];
+      const overlap = paletteOverlap(sourcePalette, candPalette);
+      const why = [`Visual match (CLIP · cosine ${Number(x.score).toFixed(3)})`];
+      if (overlap.pickedUp.length) why.push(`Picks up: ${overlap.pickedUp.slice(0, 3).join(', ')}`);
+      return {
+        dw_sku: x.dw_sku,
+        handle: x.handle || (row && row.handle) || null,
+        title: (row && row.title) || x.pattern || x.dw_sku,
+        vendor: x.vendor || (row && row.vendor) || null,
+        image_url: x.image || (row && row.image_url) || null,
+        pattern_name: x.pattern || (row && row.pattern_name) || null,
+        _score: x.score, _palette: candPalette, _dE: overlap.avgDE, _why: why
+      };
+    });
+  } catch (e) {
+    console.error('clipSimilarTop fallback →', e.message);
+    return null;
+  }
+}
+
 function pickPairs(source, candidates, k = 24) {
   const sCls = classify(parseTags(source.tags));
   const sourcePalette = paletteOf(source);
@@ -849,19 +916,29 @@ app.get('/api/similar', async (req, res) => {
     const source = await loadSource({ dw_sku, handle });
     if (!source) return res.status(404).json({ ok: false, error: 'source not found' });
 
-    // "Similar" keys on motif/material/era tags — without tags we can't compute it.
-    if (!source.tags || parseTags(source.tags).length === 0) {
-      return res.json({
-        ok: true,
-        source: { dw_sku: source.dw_sku, handle: source.handle, title: source.title, image_url: source.image_url, vendor: source.vendor },
-        similar: [],
-        reason: 'no tags on source — cannot compute similar designs'
-      });
+    const requestedK = Math.max(1, Math.min(24, parseInt(req.query.limit, 10) || 12));
+    const engine = (req.query.engine === 'clip') ? 'clip' : 'tag';
+
+    // CLIP engine (opt-in): image-embedding nearest-neighbour, no tag dependency.
+    let top = null, usedEngine = 'tag';
+    if (engine === 'clip') {
+      top = await clipSimilarTop(source, requestedK);   // null → service down / sku not embedded
+      if (top) usedEngine = 'clip';
     }
 
-    const requestedK = Math.max(1, Math.min(24, parseInt(req.query.limit, 10) || 12));
-    const candidates = await loadSimilarCandidates(source, 1200);
-    const top = pickSimilar(source, candidates, requestedK);
+    // Tag engine (default, and the fallback when CLIP misses). Keys on motif/material/era.
+    if (!top) {
+      if (!source.tags || parseTags(source.tags).length === 0) {
+        return res.json({
+          ok: true, engine: 'tag',
+          source: { dw_sku: source.dw_sku, handle: source.handle, title: source.title, image_url: source.image_url, vendor: source.vendor },
+          similar: [],
+          reason: 'no tags on source — cannot compute similar designs'
+        });
+      }
+      const candidates = await loadSimilarCandidates(source, 1200);
+      top = pickSimilar(source, candidates, requestedK);
+    }
 
     // Palette dots + paint chips, same shape as /api/pairs for UI parity.
     const sourcePalette = paletteOf(source);
@@ -882,6 +959,7 @@ app.get('/api/similar', async (req, res) => {
     res.set('Cache-Control', 'public, max-age=300');
     res.json({
       ok: true,
+      engine: usedEngine,
       source: {
         dw_sku: source.dw_sku, handle: source.handle, title: source.title,
         vendor: source.vendor, image_url: source.image_url, palette: paletteWithPaints(sourcePalette)

← 76a8e98 Design Coordinate: guard degenerate pattern labels (no more · back to Dw Pairs Well · auto-save: 2026-07-12T23:21:12 (1 files) — server.js 399cce5 →