crm: live LinkedIn refresh — incremental scrape of recent connections + message threads unioned into a live-delta the server reads per request (no full re-export needed)

c0a1f7cc4c9aa105a024daa1264b844d9046195b · 2026-08-12 18:30:58 -0700 · Steve Abrams
Files touched

M .gitignore
A scripts/li-refresh.js
M server.js
Diff

commit c0a1f7cc4c9aa105a024daa1264b844d9046195b
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Aug 12 18:30:58 2026 -0700

    crm: live LinkedIn refresh — incremental scrape of recent connections + message threads unioned into a live-delta the server reads per request (no full re-export needed)
---
 .gitignore            |  1 +
 scripts/li-refresh.js | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 server.js             |  8 ++++++--
 3 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index 289394a9..f32e0553 100644
--- a/.gitignore
+++ b/.gitignore
@@ -45,3 +45,4 @@ data/linkedin-connections.csv
 data/linkedin-follows.json
 data/linkedin-export/
 data/linkedin-graph.json
+data/linkedin-graph-live.json
diff --git a/scripts/li-refresh.js b/scripts/li-refresh.js
new file mode 100644
index 00000000..4cfde97f
--- /dev/null
+++ b/scripts/li-refresh.js
@@ -0,0 +1,50 @@
+// Incremental LinkedIn refresh — keeps the CRM green-highlight current without a full re-export.
+// Scrapes only your RECENT connections (top of the list) + RECENT message threads (cheap, no
+// full-list crash) and unions any NEW names into data/linkedin-graph-live.json. The server reads
+// graph.json + graph-live.json on every request, so new connects/messages show live after each run.
+const puppeteer = require(process.env.HOME + '/.npm-global/lib/node_modules/puppeteer');
+const fs = require('fs');
+const LIVE = process.env.HOME + '/Projects/rentv/data/linkedin-graph-live.json';
+const sleep = ms => new Promise(r => setTimeout(r, ms));
+const load = () => { try { return JSON.parse(fs.readFileSync(LIVE, 'utf8')); } catch { return { connected: [], messaged: [] }; } };
+(async () => {
+  let b; try { b = await puppeteer.connect({ browserURL: 'http://127.0.0.1:18800', defaultViewport: null }); }
+  catch (e) { console.log('NO_BROWSER ' + e.message); return; }
+  const pg = (await b.pages()).find(p => true);
+  const live = load();
+  const connected = new Set(live.connected || []), messaged = new Set(live.messaged || []);
+  const before = connected.size + messaged.size;
+
+  // 1) recent connections — default sort is "Recently added"; grab the top ~3 pages only
+  try {
+    await pg.goto('https://www.linkedin.com/mynetwork/invite-connect/connections/', { waitUntil: 'domcontentloaded', timeout: 45000 });
+    await sleep(3500);
+    for (let i = 0; i < 3; i++) {
+      const ns = await pg.evaluate(() => [...document.querySelectorAll('a[href*="/in/"]')].map(a => { const t = (a.innerText || '').trim(); return t ? t.split('\n')[0].trim() : ''; }).filter(Boolean));
+      ns.forEach(n => { if (n && n.length > 1) connected.add(n); });
+      await pg.evaluate(() => { const btn = [...document.querySelectorAll('button')].find(x => /^load more$/i.test((x.innerText || '').trim())); btn && btn.click(); });
+      await sleep(1400);
+    }
+  } catch (e) { console.log('conn-scrape ' + e.message); }
+
+  // 2) recent message threads — participant names from the conversation list (best-effort)
+  try {
+    await pg.goto('https://www.linkedin.com/messaging/', { waitUntil: 'domcontentloaded', timeout: 45000 });
+    await sleep(4000);
+    const names = await pg.evaluate(() => {
+      const out = [];
+      document.querySelectorAll('.msg-conversation-listitem, li[class*="conversation"], a[href*="/messaging/thread/"]').forEach(el => {
+        const h = el.querySelector('h3, .msg-conversation-listitem__participant-names, [class*="participant"]');
+        const t = ((h && h.innerText) || '').trim().split('\n')[0].trim();
+        if (t && t.length > 1 && !/^you\b/i.test(t)) out.push(t);
+      });
+      return out;
+    });
+    names.forEach(n => { if (n && !/designer wallcoverings|steve abrams/i.test(n)) messaged.add(n); });
+  } catch (e) { console.log('msg-scrape ' + e.message); }
+
+  fs.writeFileSync(LIVE, JSON.stringify({ connected: [...connected], messaged: [...messaged] }));
+  const after = connected.size + messaged.size;
+  console.log(`REFRESHED live-delta: connected ${connected.size}, messaged ${messaged.size} (+${after - before} new)`);
+  await b.disconnect();
+})().catch(e => { console.error('ERR ' + e.message); process.exit(0); });
diff --git a/server.js b/server.js
index 312cb3fe..4087f7ec 100644
--- a/server.js
+++ b/server.js
@@ -2402,8 +2402,12 @@ function loadLinkedInFollows() {
 function liBuildPersonSets(arr){ const full = new Set(), fl = new Set(); for (const n of (arr || [])) { const nn = liNorm(n); if (nn) full.add(nn); const f = liFirstLast(n); if (f) fl.add(f); } return { full, fl }; }
 function liFirmNorm(s){ return String(s || '').toLowerCase().replace(/\b(inc|llc|llp|lp|ltd|co|corp|corporation|company|group|the|usa|us|and|of|realty|real|estate|properties|partners|associates|brokerage|commercial|residential|holdings|international|worldwide)\b/g, ' ').replace(/[^a-z0-9]+/g, ' ').trim(); }
 function loadLinkedInGraph(){
-  let g = null;
-  try { g = JSON.parse(fs.readFileSync(path.join(DATA, 'linkedin-graph.json'), 'utf8')); } catch { /* none */ }
+  // Union the export-built baseline (linkedin-graph.json) with the live-delta from the
+  // incremental refresh scraper (linkedin-graph-live.json) so new connects/messages show live.
+  const g = { connected: [], firms: [], messaged: [], invited: [], endorsed: [] };
+  for (const fn of ['linkedin-graph.json', 'linkedin-graph-live.json']) {
+    try { const j = JSON.parse(fs.readFileSync(path.join(DATA, fn), 'utf8')); for (const k of Object.keys(g)) if (Array.isArray(j[k])) g[k] = g[k].concat(j[k]); } catch { /* none */ }
+  }
   const csvConn = loadLinkedInFollows();                       // connections.csv / follows.json (fallback + union)
   const connected = liBuildPersonSets(g && g.connected);
   for (const x of csvConn.full) connected.full.add(x); for (const x of csvConn.fl) connected.fl.add(x);
← f7b12994 crm: full LinkedIn relationship layer from the whole export · back to Rentv · crm: launchd template for durable 30-min LinkedIn live-refre e84d52d4 →