idea-board: parallel enrichment (4 concurrent) + continuous background drainer — backlog finishes in minutes

c2393df83eccc79c5bc37c5ef38f9d471527d179 · 2026-07-13 00:55:39 -0700 · Steve Abrams

Files touched

M server.js

Diff

commit c2393df83eccc79c5bc37c5ef38f9d471527d179
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Mon Jul 13 00:55:39 2026 -0700

    idea-board: parallel enrichment (4 concurrent) + continuous background drainer — backlog finishes in minutes
---
 server.js | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/server.js b/server.js
index 80d96d7..8227f64 100644
--- a/server.js
+++ b/server.js
@@ -108,18 +108,26 @@ function canon(u) {
 }
 
 let ENRICH_BUSY = false;
-async function enrichNew(ideas, cap = 6) {
+const CONCURRENCY = parseInt(process.env.ENRICH_CONCURRENCY || '4', 10); // parallel local-model calls
+async function enrichOne(idea) {
+  try { CACHE[idea.id] = { ...await ollamaEnrich(idea), enriched_at: new Date().toISOString() }; }
+  catch (e) { CACHE[idea.id] = { about: idea.title || '', fit: '?', fit_reason: 'AI enrich failed: ' + e.message, next_step: 'Skip', next_detail: '', error: true }; }
+}
+async function enrichNew(ideas, cap = 24) {
   if (ENRICH_BUSY) return; ENRICH_BUSY = true;
-  let done = 0;
   try {
-    for (const idea of ideas) {
-      if (CACHE[idea.id]) continue;
-      if (done >= cap) break;
-      try { CACHE[idea.id] = { ...await ollamaEnrich(idea), enriched_at: new Date().toISOString() }; done++; saveCache(); }
-      catch (e) { CACHE[idea.id] = { about: idea.title || '', fit: '?', fit_reason: 'AI enrich failed: ' + e.message, next_step: 'Skip', next_detail: '', error: true }; saveCache(); }
+    const todo = ideas.filter(i => !CACHE[i.id]).slice(0, cap);
+    for (let i = 0; i < todo.length; i += CONCURRENCY) {
+      await Promise.all(todo.slice(i, i + CONCURRENCY).map(enrichOne));
+      saveCache();
     }
   } finally { ENRICH_BUSY = false; }
 }
+// Background drainer: keep enriching the backlog independent of page polls.
+async function drain() {
+  try { const ideas = await collectIdeas(); if (ideas.some(i => !CACHE[i.id])) await enrichNew(ideas, 24); } catch {}
+  setTimeout(drain, 2000);
+}
 
 const server = http.createServer(async (req, res) => {
   try {
@@ -139,4 +147,4 @@ const server = http.createServer(async (req, res) => {
     res.writeHead(404); res.end('not found');
   } catch (e) { res.writeHead(500); res.end(String(e.message)); }
 });
-server.listen(PORT, () => console.log(`[slack-idea-board] http://localhost:${PORT}  channels=${CHANNELS.join(',')}  model=${MODEL}`));
+server.listen(PORT, () => { console.log(`[slack-idea-board] http://localhost:${PORT}  channels=${CHANNELS.join(',')}  model=${MODEL}`); drain(); });

← 8231daf idea-board: dedupe by URL + full pagination + Actionable/Yes · back to Slack Idea Board · 5x sweep 1: favicon/apple-touch → 204 (kill the 404 console 67e25ed →