← back to Vendor Agents Viewer
Launch infra: generic agent-host.js (per-vendor, env-parameterized) + viewer Launch/Restart/Scan wired to local pm2. Launch starts the agent, Scan runs the vendor's real scraper (innovations: 8 products live). Resolves vendor_code->scraper_id; OOM-safe (one agent per click).
1e2344ea28eead08ec0df3cdeb9dc05fcc69e95a · 2026-06-10 07:54:08 -0700 · Steve
Files touched
A agent-host.jsM public/index.htmlM server.js
Diff
commit 1e2344ea28eead08ec0df3cdeb9dc05fcc69e95a
Author: Steve <steve@designerwallcoverings.com>
Date: Wed Jun 10 07:54:08 2026 -0700
Launch infra: generic agent-host.js (per-vendor, env-parameterized) + viewer Launch/Restart/Scan wired to local pm2. Launch starts the agent, Scan runs the vendor's real scraper (innovations: 8 products live). Resolves vendor_code->scraper_id; OOM-safe (one agent per click).
---
agent-host.js | 79 ++++++++++++++++++++++++++++++++++++++
public/index.html | 8 ++--
server.js | 112 +++++++++++++++++++++++++++++++++---------------------
3 files changed, 152 insertions(+), 47 deletions(-)
diff --git a/agent-host.js b/agent-host.js
new file mode 100644
index 0000000..a6e4f8c
--- /dev/null
+++ b/agent-host.js
@@ -0,0 +1,79 @@
+/**
+ * Generic vendor-agent host. One process = one vendor, parameterized by env:
+ * AGENT_PORT, VENDOR_NAME, VENDOR_CODE, AGENT_NAME, SCRAPER_ID
+ * Serves /health (VCC + viewer health-check) and /api/scan (runs the vendor's
+ * real scraper via tsx against the ImportNewSkufromURL scrapers dir). Launched
+ * individually from the viewer's Launch button — never all at once (OOM-safe).
+ */
+const express = require('express');
+const { execFile } = require('child_process');
+const path = require('path');
+
+const app = express();
+app.use(express.json());
+
+const PORT = parseInt(process.env.AGENT_PORT || '9600', 10);
+const VENDOR = process.env.VENDOR_NAME || 'Unknown Vendor';
+const CODE = process.env.VENDOR_CODE || '';
+const AGENT = process.env.AGENT_NAME || 'Agent';
+const SCRAPER = process.env.SCRAPER_ID || '';
+const SCRAPERS_PROJECT = process.env.SCRAPERS_PROJECT ||
+ '/Users/stevestudio2/Projects/Designer-Wallcoverings/DW-Programming/ImportNewSkufromURL';
+
+let lastScan = null;
+let scanning = false;
+
+function runScraper(maxResults = 20) {
+ return new Promise((resolve) => {
+ if (!SCRAPER) return resolve({ ok: false, error: 'no SCRAPER_ID configured for this vendor' });
+ const code = `
+ (async()=>{
+ try{
+ const m = await import('./lib/scrapers/${SCRAPER}-new-products-scraper');
+ const pools = [m, (m.default && typeof m.default==='object') ? m.default : {}];
+ let fn = null;
+ for(const p of pools){
+ fn = p.scrapeNewProducts || Object.values(p).find(v=>typeof v==='function');
+ if(fn) break;
+ }
+ if(typeof m.default==='function' && !fn) fn = m.default;
+ if(typeof fn!=='function'){ console.log(JSON.stringify({ok:false,error:'no callable scraper export'})); return; }
+ const out = await fn({ maxResults: ${maxResults} });
+ console.log(JSON.stringify({ ok:true, count:(out||[]).length, sample:(out||[]).slice(0,8) }));
+ }catch(e){ console.log(JSON.stringify({ ok:false, error: String(e&&e.message||e) })); }
+ process.exit(0);
+ })();`;
+ const env = { ...process.env, SCRAPER_BROWSERBASE: process.env.SCRAPER_BROWSERBASE || '1' };
+ execFile('npx', ['tsx', '-e', code], { cwd: SCRAPERS_PROJECT, timeout: 120000, maxBuffer: 8 * 1024 * 1024, env },
+ (err, stdout) => {
+ const line = (stdout || '').trim().split('\n').filter(l => l.startsWith('{')).pop();
+ try { resolve(JSON.parse(line)); }
+ catch { resolve({ ok: false, error: 'scraper produced no JSON', raw: (stdout || '').slice(-200) }); }
+ });
+ });
+}
+
+app.get('/health', (req, res) => res.json({
+ ok: true, agent: AGENT, vendor: VENDOR, vendor_code: CODE, port: PORT,
+ scraper: SCRAPER || null, scanning, lastScan,
+}));
+
+app.get('/', (req, res) => res.send(`<!doctype html><meta charset=utf-8>
+ <body style="font-family:system-ui;background:#0f1115;color:#e8eaed;padding:30px">
+ <h2>🤖 ${AGENT} — ${VENDOR}</h2>
+ <p style="color:#9aa0aa">port ${PORT} · code ${CODE || '—'} · scraper ${SCRAPER || 'none configured'}</p>
+ <p>Last scan: ${lastScan ? `${lastScan.count} products @ ${lastScan.at}` : 'never run'}</p>
+ <button onclick="fetch('/api/scan',{method:'POST'}).then(r=>r.json()).then(j=>document.getElementById('o').textContent=JSON.stringify(j,null,1))"
+ style="background:#21262e;color:#34e2f4;border:1px solid #2a2f3a;border-radius:8px;padding:8px 14px;cursor:pointer">Run Scan</button>
+ <pre id=o style="color:#9aa0aa;white-space:pre-wrap"></pre></body>`));
+
+app.post('/api/scan', async (req, res) => {
+ if (scanning) return res.json({ ok: false, error: 'scan already in progress' });
+ scanning = true;
+ const r = await runScraper(req.body && req.body.maxResults || 20);
+ scanning = false;
+ if (r.ok) lastScan = { count: r.count, at: new Date().toISOString() };
+ res.json({ agent: AGENT, vendor: VENDOR, ...r });
+});
+
+app.listen(PORT, '0.0.0.0', () => console.log(`[agent] ${AGENT} (${VENDOR}) on :${PORT} scraper=${SCRAPER || 'none'}`));
diff --git a/public/index.html b/public/index.html
index 6a82a58..9b0254b 100644
--- a/public/index.html
+++ b/public/index.html
@@ -90,11 +90,13 @@ function render(){
</div>
${x.conflict_owner?`<div class="when" style="color:var(--conf)">\u26a0 port :${x.agent_port} reused by <b>${x.conflict_owner}</b> \u2014 agent not running</div>`:""}
<div class="when" title="${x.updated_at||''}">🕓 updated ${fmtDate(x.updated_at)}</div>
+ <div class="kv"><span>Scraper <b style="color:${x.launchable?'var(--on)':'var(--off)'}">${x.scraper_id||'none built'}</b></span></div>
<div class="btns">
${x.url?`<a class="act go" href="${x.url}" target="_blank">Open</a>`:''}
- <button class="act" onclick='act("restart",{pm2_name:${JSON.stringify(x.pm2_name)}},"restart ${x.pm2_name}")' ${x.pm2_name?'':'disabled'}>Restart</button>
- <button class="act" onclick='act("launch",{pm2_name:${JSON.stringify(x.pm2_name)}},"launch ${x.pm2_name}")' ${x.pm2_name?'':'disabled'}>Launch</button>
- <button class="act" onclick='act("scan",{vendor_code:${JSON.stringify(x.vendor_code)}},"scan ${x.vendor_code}")'>Scan</button>
+ ${x.status==='online'
+ ? `<button class="act" onclick='act("restart",{vendor_code:${JSON.stringify(x.vendor_code)}},"restart ${x.agent_name}")'>Restart</button>
+ <button class="act go" onclick='act("scan",{vendor_code:${JSON.stringify(x.vendor_code)}},"scan ${x.agent_name}")'>Scan</button>`
+ : `<button class="act go" onclick='act("launch",{vendor_code:${JSON.stringify(x.vendor_code)}},"launch ${x.agent_name}")'>Launch</button>`}
</div>
</div>`).join('')||'<p style="color:var(--mut);padding:20px">No agents match.</p>';
}
diff --git a/server.js b/server.js
index 4e087f0..a511a7d 100644
--- a/server.js
+++ b/server.js
@@ -7,10 +7,28 @@
const express = require('express');
const { Pool } = require('pg');
const { execFile } = require('child_process');
+const fs = require('fs');
+const path = require('path');
const app = express();
const PORT = process.env.PORT || 9789;
const SSH = 'my-server'; // ~/.ssh/config alias for root@45.61.58.125
+// Agents run LOCALLY on Mac2 (where the scrapers live). Launch = pm2 start agent-host.
+const SCRAPERS_DIR = '/Users/stevestudio2/Projects/Designer-Wallcoverings/DW-Programming/ImportNewSkufromURL/lib/scrapers';
+const AGENT_HOST = path.join(__dirname, 'agent-host.js');
+// vendor_code (underscores) -> scraper id (hyphens); null if no scraper file exists.
+function resolveScraper(vendorCode) {
+ if (!vendorCode) return null;
+ const cands = [vendorCode.replace(/_/g, '-'), vendorCode.replace(/_/g, ''), vendorCode];
+ for (const c of cands) {
+ if (fs.existsSync(path.join(SCRAPERS_DIR, `${c}-new-products-scraper.ts`))) return c;
+ }
+ return null;
+}
+function localExec(cmd) {
+ return new Promise((resolve) => execFile('bash', ['-lc', cmd],
+ { timeout: 60000, maxBuffer: 8 * 1024 * 1024 }, (e, out, err) => resolve({ ok: !e, out: (out || '') + (err || '') })));
+}
const pool = new Pool({
host: '127.0.0.1', port: 5432, user: 'dw_admin',
@@ -24,22 +42,23 @@ app.use(express.static(__dirname + '/public'));
// The registry's pm2_name is unreliable; resolve the ACTUAL pm2 process that
// owns each agent_port so restart/launch hit the right process.
let portCache = { at: 0, map: {} };
-function sshRaw(cmd) {
- return new Promise((resolve) => execFile('ssh', ['-o', 'ConnectTimeout=15', SSH, cmd],
- { timeout: 25000, maxBuffer: 8 * 1024 * 1024 }, (e, out) => resolve(out || '')));
-}
+// LOCAL port -> pm2 process name (agents run on Mac2). Cached 15s.
async function livePortMap() {
- if (Date.now() - portCache.at < 30000) return portCache.map;
- // port -> pid (from ss) and pid -> name (from pm2)
- const ssTxt = await sshRaw("ss -tlnp 2>/dev/null | grep -oE ':(9[0-9]{3}) +.*pid=[0-9]+'");
- const pm2Txt = await sshRaw("pm2 jlist 2>/dev/null");
+ if (Date.now() - portCache.at < 15000) return portCache.map;
+ const ls = await localExec("lsof -nP -iTCP -sTCP:LISTEN 2>/dev/null | grep -oE 'node .*:(9[0-9]{3}) ' | grep -oE ':(9[0-9]{3})' | tr -d ':' | sort -u");
+ const ports = new Set(ls.out.split('\n').map(s => s.trim()).filter(Boolean));
+ // pid->name + port->pid via lsof for node, joined with pm2
+ const lsofPid = await localExec("lsof -nP -iTCP -sTCP:LISTEN 2>/dev/null | awk '/node/{print $2\" \"$9}'");
+ const pm2j = await localExec("pm2 jlist 2>/dev/null");
const pidName = {};
- try { JSON.parse(pm2Txt).forEach(p => { pidName[String(p.pid)] = p.name; }); } catch {}
+ try { JSON.parse(pm2j.out).forEach(p => { pidName[String(p.pid)] = p.name; }); } catch {}
const map = {};
- ssTxt.split('\n').forEach(l => {
- const m = l.match(/:(9\d{3})\s.*pid=(\d+)/);
- if (m) map[m[1]] = pidName[m[2]] || ('pid:' + m[2]);
+ lsofPid.out.split('\n').forEach(l => {
+ const m = l.match(/^(\d+)\s.*:(9\d{3})$/);
+ if (m) map[m[2]] = pidName[m[1]] || ('pid:' + m[1]);
});
+ // ensure any listening port present even if pm2 name unknown
+ ports.forEach(p => { if (!map[p]) map[p] = 'listening'; });
portCache = { at: Date.now(), map };
return map;
}
@@ -65,25 +84,23 @@ app.get('/api/agents', async (req, res) => {
const slug = s => (s || '').toLowerCase().replace(/[^a-z0-9]/g, '');
const agents = rows.map(a => {
const realPm2 = a.agent_port ? live[String(a.agent_port)] : null;
- // An agent is truly ONLINE only if the live process owning its port plausibly
- // matches it. Many registry ports have been reused by unrelated apps.
- const matches = realPm2 && (
- realPm2 === a.pm2_name ||
- (a.pm2_name && slug(realPm2) === slug(a.pm2_name)) ||
- (a.vendor_name && slug(realPm2).includes(slug(a.vendor_name).slice(0, 8))) ||
- (a.agent_name && slug(realPm2).includes(slug(a.agent_name)))
- );
+ const scraperId = resolveScraper(a.vendor_code);
+ const myPm2 = 'va-' + slug(a.vendor_code); // name this viewer launches under
+ // ONLINE only if the live process owning the port is THIS viewer's agent.
+ const matches = realPm2 && (realPm2 === myPm2 || slug(realPm2) === slug(myPm2));
let status = 'offline';
if (matches) status = 'online';
- else if (realPm2) status = 'conflict'; // port reused by a different app
+ else if (realPm2) status = 'conflict'; // port reused by an unrelated app
return {
...a,
status,
- pm2_name: matches ? realPm2 : a.pm2_name,
+ scraper_id: scraperId,
+ launchable: !!scraperId,
+ pm2_name: myPm2,
registry_pm2: a.pm2_name,
conflict_owner: (!matches && realPm2) ? realPm2 : null,
- description: describe(a),
- url: a.agent_port ? `http://45.61.58.125:${a.agent_port}/` : null,
+ description: describe(a) + (scraperId ? '' : ' (no scraper built yet — launch serves health only)'),
+ url: (matches && a.agent_port) ? `http://127.0.0.1:${a.agent_port}/` : null,
};
});
const summary = {
@@ -98,36 +115,43 @@ app.get('/api/agents', async (req, res) => {
} catch (e) { res.status(500).json({ ok: false, error: e.message }); }
});
-function sshExec(cmd) {
- return new Promise((resolve) => {
- execFile('ssh', ['-o', 'ConnectTimeout=15', SSH, cmd], { timeout: 30000 },
- (err, stdout, stderr) => resolve({ ok: !err, out: (stdout || '') + (stderr || '') }));
- });
+async function vendorRow(code) {
+ const { rows } = await pool.query(
+ 'SELECT vendor_code, vendor_name, agent_name, agent_port FROM vendor_registry WHERE vendor_code=$1', [code]);
+ return rows[0];
}
+const sh = s => "'" + String(s || '').replace(/'/g, "'\\''") + "'";
-// --- restart an agent's pm2 process ---
-app.post('/api/restart', async (req, res) => {
- const name = String(req.body.pm2_name || '').replace(/[^a-zA-Z0-9_.-]/g, '');
- if (!name) return res.json({ ok: false, error: 'no pm2_name' });
- portCache.at = 0; // force port refresh next read
- const r = await sshExec(`pm2 restart ${name} --update-env 2>&1 | tail -3; pm2 describe ${name} 2>/dev/null | grep -E 'status' | head -1`);
- res.json({ ...r, name });
+// --- LAUNCH: pm2-start the generic agent host locally for this vendor ---
+app.post('/api/launch', async (req, res) => {
+ const code = String(req.body.vendor_code || '').replace(/[^a-zA-Z0-9_-]/g, '');
+ const v = await vendorRow(code);
+ if (!v || !v.agent_port) return res.json({ ok: false, error: 'vendor has no agent_port' });
+ const scraper = resolveScraper(code) || '';
+ const name = 'va-' + code.toLowerCase().replace(/[^a-z0-9]/g, '');
+ portCache.at = 0;
+ const env = `AGENT_PORT=${v.agent_port} VENDOR_NAME=${sh(v.vendor_name)} VENDOR_CODE=${sh(code)} ` +
+ `AGENT_NAME=${sh(v.agent_name || 'Agent')} SCRAPER_ID=${sh(scraper)}`;
+ const cmd = `pm2 delete ${name} >/dev/null 2>&1; ${env} pm2 start ${sh(AGENT_HOST)} --name ${name} --update-env 2>&1 | tail -2; pm2 save >/dev/null 2>&1`;
+ const r = await localExec(cmd);
+ res.json({ ...r, name, port: v.agent_port, scraper: scraper || '(none — health only)' });
});
-// --- launch (start) an agent: pm2 start by name if a process def exists, else report ---
-app.post('/api/launch', async (req, res) => {
- const name = String(req.body.pm2_name || '').replace(/[^a-zA-Z0-9_.-]/g, '');
- if (!name) return res.json({ ok: false, error: 'no pm2_name' });
+// --- RESTART: restart this vendor's local agent ---
+app.post('/api/restart', async (req, res) => {
+ const code = String(req.body.vendor_code || '').replace(/[^a-zA-Z0-9_-]/g, '');
+ const name = 'va-' + code.toLowerCase().replace(/[^a-z0-9]/g, '');
portCache.at = 0;
- const r = await sshExec(`pm2 start ${name} 2>&1 | tail -3 || echo 'no saved process def for ${name}'`);
+ const r = await localExec(`pm2 restart ${name} --update-env 2>&1 | tail -2 || echo 'not running — use Launch'`);
res.json({ ...r, name });
});
-// --- trigger a scan via the VCC proxy (agent /api/scan) ---
+// --- SCAN: hit the running agent's /api/scan (runs the real scraper) ---
app.post('/api/scan', async (req, res) => {
const code = String(req.body.vendor_code || '').replace(/[^a-zA-Z0-9_-]/g, '');
- if (!code) return res.json({ ok: false, error: 'no vendor_code' });
- const r = await sshExec(`curl -s -m 20 -X POST http://127.0.0.1:9660/api/scan/${code} 2>&1 | head -c 400`);
+ const v = await vendorRow(code);
+ if (!v || !v.agent_port) return res.json({ ok: false, error: 'no agent_port' });
+ const r = await localExec(`curl -s -m 130 -X POST http://127.0.0.1:${v.agent_port}/api/scan -H 'Content-Type: application/json' -d '{"maxResults":8}' 2>&1 | head -c 800`);
res.json(r);
});
← 913eff2 Honest agent status: resolve real pm2 name per port, flag co
·
back to Vendor Agents Viewer
·
resolveScraper alias map: 23 vendor codes mapped to existing 848ffce →