← back to All Designerwallcoverings
all.dw: async job+poll for metered live check (never 504s the proxy)
ad479fd8fc6fd4ee54d7056b8a647b017a199327 · 2026-07-27 14:39:37 -0700 · Steve Abrams
DB_AUTHORITATIVE (Kravet) tier stays fully synchronous (instant local PG read,
returns status:done inline). Browserbase tiers now fire-and-poll: the initiating
request starts the scrape WITHOUT awaiting it and returns status:pending
immediately (well under the ~60s Cloudflare/nginx proxy cap), so slow portals
(WallQuest/Romo ~63s) can no longer 504. Spend-accrual + result caching moved
into the background .then(); .catch caches an error result so a later poll
resolves cleanly and no unhandled rejection can occur. Front-end polls
livestock?...&poll=1 every poll_after_ms (max ~30 polls/~90s), renders on
status:done, falls back to the Email-Vendor copy on status:gone/exhaustion.
LIVE_WORKER made env-overridable for mocked local verification.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
M public/index.htmlM server.js
Diff
commit ad479fd8fc6fd4ee54d7056b8a647b017a199327
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Jul 27 14:39:37 2026 -0700
all.dw: async job+poll for metered live check (never 504s the proxy)
DB_AUTHORITATIVE (Kravet) tier stays fully synchronous (instant local PG read,
returns status:done inline). Browserbase tiers now fire-and-poll: the initiating
request starts the scrape WITHOUT awaiting it and returns status:pending
immediately (well under the ~60s Cloudflare/nginx proxy cap), so slow portals
(WallQuest/Romo ~63s) can no longer 504. Spend-accrual + result caching moved
into the background .then(); .catch caches an error result so a later poll
resolves cleanly and no unhandled rejection can occur. Front-end polls
livestock?...&poll=1 every poll_after_ms (max ~30 polls/~90s), renders on
status:done, falls back to the Email-Vendor copy on status:gone/exhaustion.
LIVE_WORKER made env-overridable for mocked local verification.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
public/index.html | 28 +++++++++++++--
server.js | 100 +++++++++++++++++++++++++++++++++++++++++-------------
2 files changed, 102 insertions(+), 26 deletions(-)
diff --git a/public/index.html b/public/index.html
index 153ff65..6608fab 100644
--- a/public/index.html
+++ b/public/index.html
@@ -868,6 +868,10 @@ function renderLive(pop, d) {
const when = (d.cached ? 'cached · as of ' : 'fresh · ') + esc(fmtWhen(d.as_of));
pop.innerHTML = `${dot}${bits.length ? '<div class="sline">' + bits.join(' · ') + '</div>' : ''}<div class="sline mut">${when} · <b class="lcost">${cost}</b></div>`;
}
+// Terminal responses render immediately (synchronous DB tier + rate-limit/cap/dim). A status:'pending'
+// means the slow (Browserbase) scrape is running in the background — the initiating request no longer
+// blocks on it (so it never 504s the proxy), so we poll /api/livestock…&poll=1 until status:'done'.
+const LIVE_POLL_MAX = 30; // ~90s of polling at the default 3s cadence
function fireLiveCheck(lb) {
if (lb.dataset.busy === '1') return; // client debounce — no double-fire
const pop = lb.closest('.acts, td.acts-td').querySelector('.livepop');
@@ -875,10 +879,28 @@ function fireLiveCheck(lb) {
pop.hidden = false; pop.className = 'livepop';
pop.innerHTML = '<span class="lspin">◌</span> Checking live vendor portal…';
lb.dataset.busy = '1'; const orig = lb.textContent; lb.textContent = 'Checking…';
+ const done = () => { lb.dataset.busy = ''; lb.textContent = orig; };
+ const tooLong = () => { pop.innerHTML = '<span class="sdot out">●</span> live check took too long — use Email Vendor'; done(); };
+
+ const poll = (n, afterMs) => {
+ if (n >= LIVE_POLL_MAX) return tooLong();
+ setTimeout(() => {
+ fetch('/api/livestock?sku=' + encodeURIComponent(sku) + '&live=1&poll=1')
+ .then((r) => r.json()).then((d) => {
+ if (d.status === 'pending') return poll(n + 1, d.poll_after_ms || 3000);
+ if (d.status === 'gone') return tooLong();
+ renderLive(pop, d); done(); // status:'done' (or any terminal shape)
+ })
+ .catch(() => { pop.innerHTML = '<span class="sdot out">●</span> Live check failed — try again'; done(); });
+ }, afterMs || 3000);
+ };
+
fetch('/api/livestock?sku=' + encodeURIComponent(sku) + '&live=1')
- .then((r) => r.json()).then((d) => renderLive(pop, d))
- .catch(() => { pop.innerHTML = '<span class="sdot out">●</span> Live check failed — try again'; })
- .finally(() => { lb.dataset.busy = ''; lb.textContent = orig; });
+ .then((r) => r.json()).then((d) => {
+ if (d.status === 'pending') return poll(0, d.poll_after_ms || 3000);
+ renderLive(pop, d); done(); // terminal on first hit (DB tier / rate-limit / cap / dim)
+ })
+ .catch(() => { pop.innerHTML = '<span class="sdot out">●</span> Live check failed — try again'; done(); });
}
// ── Email Vendor — stock & price inquiry compose (EVERY vendor; Gmail DRAFT only) ──────
diff --git a/server.js b/server.js
index 2eafea5..696bf91 100644
--- a/server.js
+++ b/server.js
@@ -170,7 +170,7 @@ setInterval(() => {
}, 10 * 60 * 1000).unref();
const coverageFor = (vendor) => COVERAGE.get(String(vendor || '').trim().toLowerCase()) || null;
const liveAdapter = (vendor) => { const e = coverageFor(vendor); return (e && e.live_capable && e.adapter) ? e.adapter : null; };
-const LIVE_WORKER = path.join(__dirname, 'scripts', 'live-scrape.js');
+const LIVE_WORKER = process.env.LIVE_WORKER || path.join(__dirname, 'scripts', 'live-scrape.js');
const LIVE_CACHE_MS = 10 * 60 * 1000; // repeat clicks within 10 min = cached, $0
const LIVE_MAX_INFLIGHT = 2; // global burst cap on concurrent portal scrapes
const LIVE_IP_MAX = 6, LIVE_IP_WIN_MS = 60 * 1000; // per-IP: ≤6 live calls / 60s
@@ -1095,30 +1095,78 @@ async function handleLiveStock(req, res, u) {
reason: (cov && cov.reason) || 'live check unavailable for this vendor', cost_usd: 0 });
}
const key = sku.toUpperCase();
+ const isPoll = u.searchParams.get('poll') === '1';
// cache: repeat within the 10-min window returns the cached live result, $0.
+ // (Applies to BOTH tiers and BOTH initiate + poll — a fresh cache hit is always "done".)
const hit = liveCache.get(key);
if (hit && (Date.now() - hit.at) < LIVE_CACHE_MS) {
- return sendJSON(res, 200, { ...hit.result, cached: true, cost_usd: 0, cache_age_s: Math.round((Date.now() - hit.at) / 1000) });
+ return sendJSON(res, 200, { ...hit.result, status: 'done', cached: true, cost_usd: 0, cache_age_s: Math.round((Date.now() - hit.at) / 1000) });
}
- // coalesce: a scrape already running for this SKU → ride it (no second bill).
+
+ // ── DB_AUTHORITATIVE tier stays fully SYNCHRONOUS ──────────────────────────
+ // It's an instant local Postgres read (no Browserbase session, $0) — zero proxy risk, so we
+ // keep the original await-inline behavior and return the full result with status:'done'.
+ // The async job+poll machinery below is only for the SLOW (Browserbase) tiers that can 504.
+ if (cov.tier === 'DB_AUTHORITATIVE') {
+ // coalesce: a read already running for this SKU → ride it (no second call).
+ if (liveInflight.has(key)) {
+ try { const r = await liveInflight.get(key); return sendJSON(res, 200, { ...r, status: 'done', cached: true, coalesced: true, cost_usd: 0 }); }
+ catch { return sendJSON(res, 200, { ok: false, live: true, live_available: true, sku, reason: 'live scrape failed', cost_usd: 0, status: 'done' }); }
+ }
+ // per-IP rate window (DB tier is $-cap exempt but still rate-limited)
+ const ipDb = (String(req.headers['x-forwarded-for'] || req.socket.remoteAddress || '')).split(',')[0].trim();
+ const nowDb = Date.now();
+ const hitsDb = (liveIpHits.get(ipDb) || []).filter((t) => nowDb - t < LIVE_IP_WIN_MS);
+ if (hitsDb.length >= LIVE_IP_MAX) {
+ return sendJSON(res, 429, { ok: false, live: true, rate_limited: true, sku,
+ reason: 'too many live checks — wait a moment', retry_after_s: Math.ceil((LIVE_IP_WIN_MS - (nowDb - hitsDb[0])) / 1000), cost_usd: 0 });
+ }
+ if (liveInflight.size >= LIVE_MAX_INFLIGHT) {
+ return sendJSON(res, 429, { ok: false, live: true, rate_limited: true, sku,
+ reason: 'live-check queue full — try again shortly', retry_after_s: 8, cost_usd: 0 });
+ }
+ hitsDb.push(nowDb); liveIpHits.set(ipDb, hitsDb);
+ const pDb = runLiveScrape(sku, adapter, cov.catalog_table).finally(() => liveInflight.delete(key));
+ liveInflight.set(key, pDb);
+ let resultDb;
+ try { resultDb = await pDb; } catch (e) { return sendJSON(res, 200, { ok: false, live: true, live_available: true, sku, reason: 'live scrape error: ' + e.message, cost_usd: 0, status: 'done' }); }
+ // DB tier opens no billable session (cost_usd stays 0) — still surface the running day total.
+ if (resultDb.cost_usd > 0) addDaySpend(resultDb.cost_usd);
+ resultDb.day_spend_usd = +daySpendUSD().toFixed(4);
+ resultDb.day_cap_usd = DAILY_LIVE_CAP_USD;
+ resultDb.status = 'done';
+ if (resultDb.ok) liveCache.set(key, { result: resultDb, at: Date.now() });
+ return sendJSON(res, 200, resultDb);
+ }
+
+ // ── Browserbase tiers — ASYNC job + poll (never block the proxy) ───────────
+ // The scrape runs in the background; the initiating request returns immediately with
+ // status:'pending'. The client polls with &poll=1 until status:'done' (or gives up).
+
+ // POLL request: report on an in-flight scrape without starting a new one.
+ if (isPoll) {
+ if (liveInflight.has(key)) return sendJSON(res, 200, { status: 'pending', sku, poll_after_ms: 3000, live: true });
+ // no fresh cache (handled above) and nothing in flight → the result is gone; ask again.
+ return sendJSON(res, 200, { status: 'gone', sku, reason: 'no live result — click Check Live again', live: true });
+ }
+
+ // INITIATE request.
+ // coalesce: a scrape already running for this SKU → tell the client to poll (no new bill).
if (liveInflight.has(key)) {
- try { const r = await liveInflight.get(key); return sendJSON(res, 200, { ...r, cached: true, coalesced: true, cost_usd: 0 }); }
- catch { return sendJSON(res, 200, { ok: false, live: true, live_available: true, sku, reason: 'live scrape failed', cost_usd: 0 }); }
+ return sendJSON(res, 200, { status: 'pending', sku, poll_after_ms: 3000, live: true });
}
- // ── DAILY $ KILL-SWITCH — the TOTAL ceiling the rate limits don't provide ──
- // Checked AFTER cache/coalesce (those are free) but BEFORE any new billable session. When the
- // day's summed spend hits the cap, hard-stop at $0 — no Browserbase session is ever opened.
- // DB_AUTHORITATIVE tier is EXEMPT from the $-cap ONLY (it opens no billable session — a pure
- // local Postgres read); the per-IP rate limit and inflight cap below still apply to it.
+ // ── DAILY $ KILL-SWITCH — the TOTAL ceiling the rate limits don't provide (TERMINAL) ──
+ // Checked BEFORE any new billable session. When the day's summed spend hits the cap,
+ // hard-stop at $0 — no Browserbase session is ever opened.
const daySpend = daySpendUSD();
- if (cov.tier !== 'DB_AUTHORITATIVE' && daySpend >= DAILY_LIVE_CAP_USD) {
+ if (daySpend >= DAILY_LIVE_CAP_USD) {
return sendJSON(res, 200, { ok: false, live: true, live_available: false, sku,
vendor: row ? row.vendor : null, tier: cov ? cov.tier : null,
reason: `daily live-stock budget reached ($${daySpend.toFixed(2)}/$${DAILY_LIVE_CAP_USD.toFixed(2)})`,
day_spend_usd: +daySpend.toFixed(4), day_cap_usd: DAILY_LIVE_CAP_USD, cost_usd: 0 });
}
- // per-IP rate window
+ // per-IP rate window (TERMINAL)
const ip = (String(req.headers['x-forwarded-for'] || req.socket.remoteAddress || '')).split(',')[0].trim();
const now = Date.now();
const hits = (liveIpHits.get(ip) || []).filter((t) => now - t < LIVE_IP_WIN_MS);
@@ -1126,24 +1174,30 @@ async function handleLiveStock(req, res, u) {
return sendJSON(res, 429, { ok: false, live: true, rate_limited: true, sku,
reason: 'too many live checks — wait a moment', retry_after_s: Math.ceil((LIVE_IP_WIN_MS - (now - hits[0])) / 1000), cost_usd: 0 });
}
- // global burst cap on concurrent portal sessions (protects spend + the vendor portal)
+ // global burst cap on concurrent portal sessions (protects spend + the vendor portal) (TERMINAL)
if (liveInflight.size >= LIVE_MAX_INFLIGHT) {
return sendJSON(res, 429, { ok: false, live: true, rate_limited: true, sku,
reason: 'live-check queue full — try again shortly', retry_after_s: 8, cost_usd: 0 });
}
hits.push(now); liveIpHits.set(ip, hits);
- const p = runLiveScrape(sku, adapter, cov.catalog_table).finally(() => liveInflight.delete(key));
+ // Start the scrape but DO NOT await it in the handler — the request returns immediately so it
+ // can never 504 through the ~60s proxy cap. Spend-accrual + caching happen in the background
+ // .then() (moved out of the request path). The .catch caches an error result so a later poll
+ // resolves cleanly, and guarantees no unhandled promise rejection.
+ const p = runLiveScrape(sku, adapter, cov.catalog_table)
+ .then((r) => {
+ if (r.cost_usd > 0) addDaySpend(r.cost_usd);
+ r.day_spend_usd = +daySpendUSD().toFixed(4); r.day_cap_usd = DAILY_LIVE_CAP_USD;
+ liveCache.set(key, { result: { ...r, status: 'done' }, at: Date.now() });
+ return r;
+ })
+ .catch((e) => {
+ liveCache.set(key, { result: { ok: false, live: true, live_available: true, sku, reason: 'live scrape error: ' + e.message, cost_usd: 0, status: 'done' }, at: Date.now() });
+ })
+ .finally(() => { liveInflight.delete(key); });
liveInflight.set(key, p);
- let result;
- try { result = await p; } catch (e) { return sendJSON(res, 200, { ok: false, live: true, live_available: true, sku, reason: 'live scrape error: ' + e.message, cost_usd: 0 }); }
- // Accrue any real spend (a session may bill even on a failed parse) toward the daily ceiling,
- // then surface the running day total + cap on every metered response.
- if (result.cost_usd > 0) addDaySpend(result.cost_usd);
- result.day_spend_usd = +daySpendUSD().toFixed(4);
- result.day_cap_usd = DAILY_LIVE_CAP_USD;
- if (result.ok) liveCache.set(key, { result, at: Date.now() });
- return sendJSON(res, 200, result);
+ return sendJSON(res, 200, { status: 'pending', sku, poll_after_ms: 3000, live: true });
}
// ── /api/discontinued?sku=<sku> — resolve a dead/archived SKU + link its live successor ──
← fde98d3 auto-save: 2026-07-23T12:20:57 (1 files) — server.js
·
back to All Designerwallcoverings
·
all.dw: raise live-scrape worker timeout 52s->75s (async/pol 76b0372 →