`lib/hotspot-providers/gemini.js`

87 lines
// HOTSPOT PROVIDER: gemini — Vision-locate products inside a generated room-setting
// image so the UI can put a shoppable hotspot ON each actual piece. Uses Gemini 2.5
// Flash bounding-box detection (returns [ymin,xmin,ymax,xmax] normalized 0-1000). Any
// product the model can't confidently place is simply omitted — the frontend renders
// those as edge chips so nothing becomes unselectable. Cheap: one Flash text call per
// scene. Selected via HOTSPOT_PROVIDER=gemini (default) in lib/hotspots.js.
const fs = require('fs');

const MODEL = 'gemini-2.5-flash';
const COST_PER_CALL = 0.001; // ~1 image + small JSON out on Flash; shown to Steve

function buildPrompt(products) {
  const list = products.map((p, i) => `${i + 1}. ${p.title}`).join('\n');
  return [
    'This is a photograph of a furnished room. Below is a numbered list of the products that appear in it.',
    'For EACH product you can clearly locate, return its bounding box.',
    'Respond with ONLY a compact JSON array, no prose, no code fence:',
    '[{"i": <product number>, "box": [ymin, xmin, ymax, xmax]}]',
    'Coordinates are normalized 0-1000 (y = top→bottom, x = left→right).',
    'Omit any product you cannot confidently locate. Products:',
    list,
  ].join('\n');
}

// Intersection-over-union of two %-boxes — used to reject a new hotspot that lands
// on the same visual object as one already accepted (lookalike products the model
// tagged to the single instance it could actually see in the scene).
function iou(a, b) {
  const ix = Math.max(0, Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x));
  const iy = Math.max(0, Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y));
  const inter = ix * iy;
  return inter ? inter / (a.w * a.h + b.w * b.h - inter) : 0;
}

function toHotspots(raw, products) {
  const seen = new Set();
  const hotspots = [];
  for (const d of raw) {
    const idx = Number(d.i) - 1;
    const p = products[idx];
    if (!p || seen.has(p.id) || !Array.isArray(d.box) || d.box.length !== 4) continue;
    let [ymin, xmin, ymax, xmax] = d.box.map(Number);
    const cl = (n) => Math.max(0, Math.min(1000, n || 0));
    ymin = cl(ymin); xmin = cl(xmin); ymax = cl(ymax); xmax = cl(xmax);
    if (xmax <= xmin || ymax <= ymin) continue;
    const box = { x: xmin / 10, y: ymin / 10, w: (xmax - xmin) / 10, h: (ymax - ymin) / 10 };
    if (hotspots.some((h) => iou(h.box, box) > 0.45)) continue; // don't stack on one object
    seen.add(p.id);
    hotspots.push({ id: p.id, title: p.title, price: p.sale_price ?? p.price, image_url: p.image_url, advertiser: p.advertiser, box });
  }
  return hotspots;
}

// products: [{id, title, image_url, price, sale_price, advertiser}]
// imagePath: absolute path to the generated PNG on disk
async function locateProducts(imagePath, products = []) {
  const key = process.env.GEMINI_API_KEY;
  if (!key || !products.length) return { hotspots: [], cost: 0 };
  let b64;
  try { b64 = fs.readFileSync(imagePath).toString('base64'); }
  catch { return { hotspots: [], cost: 0 }; }

  const prompt = buildPrompt(products);
  let j;
  try {
    const res = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${MODEL}:generateContent?key=${key}`, {
      method: 'POST', headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        contents: [{ parts: [{ inlineData: { mimeType: 'image/png', data: b64 } }, { text: prompt }] }],
        generationConfig: { temperature: 0, responseMimeType: 'application/json' },
      }),
    });
    j = await res.json();
  } catch (e) { return { hotspots: [], cost: 0, error: e.message }; }
  if (j.error) return { hotspots: [], cost: 0, error: j.error.message };

  const text = (j.candidates && j.candidates[0] && j.candidates[0].content
    && j.candidates[0].content.parts || []).map((p) => p.text || '').join('');
  let raw;
  try { raw = JSON.parse(text); } catch { raw = []; }
  if (!Array.isArray(raw)) raw = [];

  return { hotspots: toHotspots(raw, products), cost: COST_PER_CALL };
}

module.exports = { locateProducts, COST_PER_CALL };