`agent-host.js`

93 lines
/**
 * Generic vendor-agent host. One process = one vendor, parameterized by env:
 *   AGENT_PORT, VENDOR_NAME, VENDOR_CODE, AGENT_NAME, SCRAPER_ID
 * Serves /health (VCC + viewer health-check) and /api/scan (runs the vendor's
 * real scraper via tsx against the ImportNewSkufromURL scrapers dir). Launched
 * individually from the viewer's Launch button — never all at once (OOM-safe).
 */
const express = require('express');
const { execFile } = require('child_process');
const path = require('path');

const app = express();
app.use(express.json());

const PORT = parseInt(process.env.AGENT_PORT || '9600', 10);
const VENDOR = process.env.VENDOR_NAME || 'Unknown Vendor';
const CODE = process.env.VENDOR_CODE || '';
const AGENT = process.env.AGENT_NAME || 'Agent';
const SCRAPER = process.env.SCRAPER_ID || '';
const SCRAPERS_PROJECT = process.env.SCRAPERS_PROJECT ||
  '/Users/macstudio3/Projects/Designer-Wallcoverings/DW-Programming/ImportNewSkufromURL';

let lastScan = null;
let scanning = false;

function runScraper(maxResults = 20) {
  return new Promise((resolve) => {
    if (!SCRAPER) return resolve({ ok: false, error: 'no SCRAPER_ID configured for this vendor' });
    const code = `
      (async()=>{
        try{
          const m = await import('./lib/scrapers/${SCRAPER}-new-products-scraper');
          const pools = [m, (m.default && typeof m.default==='object') ? m.default : {}];
          let fn = null;
          for(const p of pools){
            fn = p.scrapeNewProducts || Object.values(p).find(v=>typeof v==='function');
            if(fn) break;
          }
          if(typeof m.default==='function' && !fn) fn = m.default;
          if(typeof fn!=='function'){ console.log(JSON.stringify({ok:false,error:'no callable scraper export'})); return; }
          const out = await fn({ maxResults: ${maxResults} });
          // scrapers return either a bare array or a {vendor, products} wrapper
          const arr = Array.isArray(out) ? out : (out && Array.isArray(out.products) ? out.products : []);
          console.log(JSON.stringify({ ok:true, count:arr.length, sample:arr.slice(0,8) }));
        }catch(e){ console.log(JSON.stringify({ ok:false, error: String(e&&e.message||e) })); }
        process.exit(0);
      })();`;
    const env = { ...process.env, SCRAPER_BROWSERBASE: process.env.SCRAPER_BROWSERBASE || '1' };
    execFile('npx', ['tsx', '-e', code], { cwd: SCRAPERS_PROJECT, timeout: 120000, maxBuffer: 8 * 1024 * 1024, env },
      (err, stdout) => {
        const line = (stdout || '').trim().split('\n').filter(l => l.startsWith('{')).pop();
        try { resolve(JSON.parse(line)); }
        catch { resolve({ ok: false, error: 'scraper produced no JSON', raw: (stdout || '').slice(-200) }); }
      });
  });
}

app.get('/health', (req, res) => res.json({
  ok: true, agent: AGENT, vendor: VENDOR, vendor_code: CODE, port: PORT,
  scraper: SCRAPER || null, scanning, lastScan,
}));

app.get('/', (req, res) => res.send(`<!doctype html><meta charset=utf-8>
  <body style="font-family:system-ui;background:#0f1115;color:#e8eaed;padding:30px">
  <h2>🤖 ${AGENT} — ${VENDOR}</h2>
  <p style="color:#9aa0aa">port ${PORT} · code ${CODE || '—'} · scraper ${SCRAPER || 'none configured'}</p>
  <p>Last scan: ${lastScan ? `${lastScan.count} products @ ${lastScan.at}` : 'never run'}</p>
  <button onclick="fetch('/api/scan',{method:'POST'}).then(r=>r.json()).then(j=>document.getElementById('o').textContent=JSON.stringify(j,null,1))"
    style="background:#21262e;color:#34e2f4;border:1px solid #2a2f3a;border-radius:8px;padding:8px 14px;cursor:pointer">Run Scan</button>
  <pre id=o style="color:#9aa0aa;white-space:pre-wrap"></pre></body>`));

app.post('/api/scan', async (req, res) => {
  if (scanning) return res.json({ ok: false, error: 'scan already in progress' });
  scanning = true;
  const r = await runScraper(req.body && req.body.maxResults || 20);
  scanning = false;
  if (r.ok) lastScan = { count: r.count, at: new Date().toISOString() };
  res.json({ agent: AGENT, vendor: VENDOR, ...r });
});

// Bind with EADDRINUSE backoff: on a pm2 restart the prior instance may not have
// released the listen socket yet. Without this, the new instance hot-loops on
// EADDRINUSE and spams the error log (241MB observed for va-muralsource). Throttle
// to one retry / 2s so the old socket has time to free instead of millions of lines.
const server = app.listen(PORT, '0.0.0.0', () =>
  console.log(`[agent] ${AGENT} (${VENDOR}) on :${PORT} scraper=${SCRAPER || 'none'}`));
server.on('error', (err) => {
  if (err.code === 'EADDRINUSE') {
    console.error(`[agent] :${PORT} busy (prior instance releasing), retry in 2s`);
    setTimeout(() => { try { server.close(); } catch {} app.listen(PORT, '0.0.0.0'); }, 2000);
  } else { throw err; }
});