`lib/hotspot-providers/openai.js`

101 lines
// HOTSPOT PROVIDER: openai — FALLBACK for when the Gemini prepay is depleted
// (TK-11260, follow-on to the TK-10402 SCENE_PROVIDER fallback pattern). Same
// contract as hotspot-providers/gemini.js: vision-locate each product inside a
// generated room-setting image and return a normalized %-box per product so the
// frontend can render a shoppable hotspot ON the actual piece. Uses gpt-5.2 (vision)
// via the Responses API — cost ~$0.003-0.006/room (image + short JSON out), a
// little pricier than Gemini Flash's ~$0.001 but funded when Gemini isn't.
// Selected via HOTSPOT_PROVIDER=openai in lib/hotspots.js.
const fs = require('fs');

const MODEL = 'gpt-5.2';
const COST_PER_CALL = 0.005; // conservative estimate for one image + small JSON out; shown to Steve

function buildPrompt(products) {
  const list = products.map((p, i) => `${i + 1}. ${p.title}`).join('\n');
  return [
    'This is a photograph of a furnished room. Below is a numbered list of the products that appear in it.',
    'For EACH product you can clearly locate, return its bounding box.',
    'Respond with ONLY a compact JSON array, no prose, no code fence, no markdown:',
    '[{"i": <product number>, "box": [ymin, xmin, ymax, xmax]}]',
    'Coordinates are normalized 0-1000 (y = top→bottom, x = left→right), relative to the full image.',
    'Omit any product you cannot confidently locate. Products:',
    list,
  ].join('\n');
}

function iou(a, b) {
  const ix = Math.max(0, Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x));
  const iy = Math.max(0, Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y));
  const inter = ix * iy;
  return inter ? inter / (a.w * a.h + b.w * b.h - inter) : 0;
}

function toHotspots(raw, products) {
  const seen = new Set();
  const hotspots = [];
  for (const d of raw) {
    const idx = Number(d.i) - 1;
    const p = products[idx];
    if (!p || seen.has(p.id) || !Array.isArray(d.box) || d.box.length !== 4) continue;
    let [ymin, xmin, ymax, xmax] = d.box.map(Number);
    const cl = (n) => Math.max(0, Math.min(1000, n || 0));
    ymin = cl(ymin); xmin = cl(xmin); ymax = cl(ymax); xmax = cl(xmax);
    if (xmax <= xmin || ymax <= ymin) continue;
    const box = { x: xmin / 10, y: ymin / 10, w: (xmax - xmin) / 10, h: (ymax - ymin) / 10 };
    if (hotspots.some((h) => iou(h.box, box) > 0.45)) continue;
    seen.add(p.id);
    hotspots.push({ id: p.id, title: p.title, price: p.sale_price ?? p.price, image_url: p.image_url, advertiser: p.advertiser, box });
  }
  return hotspots;
}

function extractJsonArray(text) {
  if (!text) return [];
  // Strip a stray code fence if the model ignores the "no code fence" instruction.
  const cleaned = text.trim().replace(/^```(?:json)?/i, '').replace(/```$/, '').trim();
  try {
    const parsed = JSON.parse(cleaned);
    return Array.isArray(parsed) ? parsed : [];
  } catch { return []; }
}

async function locateProducts(imagePath, products = []) {
  const key = process.env.OPENAI_API_KEY;
  if (!key || !products.length) return { hotspots: [], cost: 0 };
  let b64;
  try { b64 = fs.readFileSync(imagePath).toString('base64'); }
  catch { return { hotspots: [], cost: 0 }; }

  const prompt = buildPrompt(products);
  let j;
  try {
    const res = await fetch('https://api.openai.com/v1/responses', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },
      body: JSON.stringify({
        model: MODEL,
        input: [{
          role: 'user',
          content: [
            { type: 'input_text', text: prompt },
            { type: 'input_image', image_url: `data:image/png;base64,${b64}` },
          ],
        }],
      }),
    });
    j = await res.json();
  } catch (e) { return { hotspots: [], cost: 0, error: e.message }; }
  if (j.error) return { hotspots: [], cost: 0, error: j.error.message };

  const text = (j.output || [])
    .flatMap((o) => (o.content || []))
    .map((c) => c.text || '')
    .join('');
  const raw = extractJsonArray(text);

  return { hotspots: toHotspots(raw, products), cost: COST_PER_CALL };
}

module.exports = { locateProducts, COST_PER_CALL };