vendors: robust auto-updating social pipeline — weekly social-refresh (IG API + local LinkedIn OG harvest→prod sync + contact regen + heartbeat) & standing export-ingest checker (watcher-only)

3bf588c3cea9b985dc4826c410296d1c7a591f57 · 2026-08-31 14:15:49 -0700 · Steve Abrams
Files touched

M .gitignore
A scripts/ingest-linkedin-exports.mjs
A scripts/social-refresh.mjs
Diff

commit 3bf588c3cea9b985dc4826c410296d1c7a591f57
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Mon Aug 31 14:15:49 2026 -0700

    vendors: robust auto-updating social pipeline — weekly social-refresh (IG API + local LinkedIn OG harvest→prod sync + contact regen + heartbeat) & standing export-ingest checker (watcher-only)
---
 .gitignore                          |  2 +
 scripts/ingest-linkedin-exports.mjs | 56 ++++++++++++++++++++++++
 scripts/social-refresh.mjs          | 87 +++++++++++++++++++++++++++++++++++++
 3 files changed, 145 insertions(+)

diff --git a/.gitignore b/.gitignore
index 76d3d4e..9f2e5ba 100644
--- a/.gitignore
+++ b/.gitignore
@@ -86,3 +86,5 @@ data/linkedin-network-firms.json
 data/vendor-linkedin-follows.json
 data/vendor-contacts.json
 data/vendor-contacts.html
+data/linkedin-ingest-state.json
+data/social-refresh-latest.json
diff --git a/scripts/ingest-linkedin-exports.mjs b/scripts/ingest-linkedin-exports.mjs
new file mode 100644
index 0000000..2a4064d
--- /dev/null
+++ b/scripts/ingest-linkedin-exports.mjs
@@ -0,0 +1,56 @@
+// Standing ingest checker — run periodically by launchd. Idempotently ingests the
+// NEWEST LinkedIn export found on Desktop/Downloads: Company Follows (→ follows +
+// slug-resolve for NEW firms) and Connections (→ contact sheet). Tracks last-ingested
+// mtime so re-runs are no-ops until you drop a fresher export. Watcher-only (no reminders).
+//   node scripts/ingest-linkedin-exports.mjs
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { execFileSync } from 'node:child_process';
+
+const ROOT = path.join(path.dirname(new URL(import.meta.url).pathname), '..');
+const STATE = path.join(ROOT, 'data', 'linkedin-ingest-state.json');
+const roots = [path.join(os.homedir(), 'Downloads'), path.join(os.homedir(), 'Desktop')];
+const log = (...a) => console.log(new Date().toISOString(), ...a);
+
+function findNewest(nameRe) {
+  let best = null;
+  for (const r of roots) {
+    let stack = [{ d: r, depth: 0 }];
+    while (stack.length) {
+      const { d, depth } = stack.pop();
+      let ents; try { ents = fs.readdirSync(d, { withFileTypes: true }); } catch { continue; }
+      for (const e of ents) {
+        const p = path.join(d, e.name);
+        if (e.isDirectory() && depth < 3) stack.push({ d: p, depth: depth + 1 });
+        else if (e.isFile() && nameRe.test(e.name)) {
+          const m = fs.statSync(p).mtimeMs;
+          if (!best || m > best.mtime) best = { path: p, mtime: m };
+        }
+      }
+    }
+  }
+  return best;
+}
+
+const state = (() => { try { return JSON.parse(fs.readFileSync(STATE, 'utf8')); } catch { return {}; } })();
+const follows = findNewest(/^company follows\.csv$/i) || findNewest(/company.?follows.*\.csv$/i);
+const conns = findNewest(/^connections\.csv$/i);
+let did = false;
+
+if (follows && follows.mtime > (state.followsMtime || 0)) {
+  log('Ingesting follows:', follows.path);
+  execFileSync('node', [path.join(ROOT, 'scripts', 'import-linkedin-follows.mjs'), follows.path], { stdio: 'inherit' });
+  execFileSync('node', [path.join(ROOT, 'scripts', 'resolve-follows.mjs')], { stdio: 'inherit' }); // resolve new slugs
+  state.followsMtime = follows.mtime; did = true;
+} else log('follows: up to date');
+
+if (conns && conns.mtime > (state.connsMtime || 0)) {
+  log('Ingesting connections:', conns.path);
+  execFileSync('node', [path.join(ROOT, 'scripts', 'import-linkedin-connections.mjs'), conns.path], { stdio: 'inherit' });
+  execFileSync('node', [path.join(ROOT, 'scripts', 'build-vendor-contact-sheet.mjs')], { stdio: 'inherit' });
+  state.connsMtime = conns.mtime; did = true;
+} else log('connections: up to date');
+
+fs.writeFileSync(STATE, JSON.stringify({ ...state, lastCheck: new Date().toISOString() }, null, 2));
+log(did ? 'INGESTED new export(s).' : 'nothing new.');
diff --git a/scripts/social-refresh.mjs b/scripts/social-refresh.mjs
new file mode 100644
index 0000000..7699714
--- /dev/null
+++ b/scripts/social-refresh.mjs
@@ -0,0 +1,87 @@
+// Weekly social-refresh orchestrator — keeps the vendor social pull CURRENT.
+//   1. IG vendor posts  → POST prod /api/vendors/posts/refresh (official Meta Business Discovery)
+//   2. LinkedIn previews → gentle LOCAL public-OG harvest of resolved slugs (protects Kamatera's IP),
+//                          then scp the cache to prod so the live panel shows fresh previews
+//   3. Vendor contact sheet → local regen (PII stays local)
+//   4. Freshness heartbeat → data/latest.json (PASS/WARN/FAIL for the fleet-health rollup)
+// Reference/attribution-amplify only; read-only public GETs + official API. No posting, no PII to prod.
+//   node scripts/social-refresh.mjs
+import fs from 'node:fs';
+import path from 'node:path';
+import { execFileSync } from 'node:child_process';
+
+const ROOT = path.join(path.dirname(new URL(import.meta.url).pathname), '..');
+const DATA = path.join(ROOT, 'data');
+const sleep = ms => new Promise(r => setTimeout(r, ms));
+const env = k => { try { return (fs.readFileSync(path.join(ROOT, '.env'), 'utf8').match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/^['"]|['"]$/g, ''); } catch { return null; } };
+const log = (...a) => console.log(new Date().toISOString(), ...a);
+const steps = {};
+
+// ── 1. IG vendor posts — official API, run against PROD so its cache updates ──
+async function refreshIG() {
+  const U = env('MCC_USER') || 'admin', P = env('MCC_PASS') || '';
+  const auth = 'Basic ' + Buffer.from(`${U}:${P}`).toString('base64');
+  try {
+    const r = await fetch('https://marketing.designerwallcoverings.com/api/vendors/posts/refresh',
+      { method: 'POST', headers: { authorization: auth, 'content-type': 'application/json' }, body: '{}' });
+    const j = await r.json().catch(() => ({}));
+    steps.ig = { ok: r.ok, ...j };
+    log('IG refresh:', r.status, JSON.stringify(j).slice(0, 120));
+  } catch (e) { steps.ig = { ok: false, error: e.message }; log('IG refresh FAILED', e.message); }
+}
+
+// ── 2. LinkedIn previews — gentle LOCAL OG harvest of RESOLVED slugs only ──
+async function ogPull(slug) {
+  try {
+    const r = await fetch(`https://www.linkedin.com/company/${slug}/`, { redirect: 'manual', headers: {
+      'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36', accept: 'text/html' } });
+    if (r.status === 999 || r.status === 429) return { wall: true };
+    if (!r.ok) return { status: r.status };
+    const h = await r.text();
+    const g = re => (h.match(re) || [])[1] || null;
+    return { status: r.status, title: g(/<meta property="og:title" content="([^"]+)"/i),
+      thumb: g(/<meta property="og:image" content="([^"]+)"/i), description: g(/<meta property="og:description" content="([^"]+)"/i) };
+  } catch (e) { return { error: e.message }; }
+}
+async function refreshLinkedIn() {
+  const readJson = f => { try { return JSON.parse(fs.readFileSync(path.join(DATA, f), 'utf8')); } catch { return null; } };
+  const vmap = readJson('vendor-linkedin.json'); const follows = readJson('vendor-linkedin-follows.json');
+  const slugs = new Set();
+  for (const a of (vmap?.accounts || [])) if (a.slug && a.verified !== false) slugs.add(a.slug);
+  for (const a of (follows?.accounts || [])) if (a.slug && a.verified === true) slugs.add(a.slug);
+  const cacheFile = path.join(DATA, 'vendor-linkedin-cache.json');
+  let cache = {}; try { cache = JSON.parse(fs.readFileSync(cacheFile, 'utf8')); } catch {}
+  let ok = 0, fail = 0, consecWall = 0;
+  for (const slug of slugs) {
+    const og = await ogPull(slug); await sleep(1600);
+    if (og.wall) { if (++consecWall >= 8) { log('LinkedIn rate-walled — stopping early'); break; } continue; }
+    consecWall = 0;
+    if (og.title) { cache[slug] = { og: { title: og.title, thumb: og.thumb, description: og.description }, status: og.status, fetchedAt: new Date().toISOString(), error: null }; ok++; }
+    else { cache[slug] = { ...(cache[slug] || {}), status: og.status || null, fetchedAt: new Date().toISOString(), error: 'no og' }; fail++; }
+    if ((ok + fail) % 20 === 0) fs.writeFileSync(cacheFile, JSON.stringify(cache, null, 2));
+  }
+  fs.writeFileSync(cacheFile, JSON.stringify(cache, null, 2));
+  steps.linkedin = { ok, fail, slugs: slugs.size };
+  log(`LinkedIn harvest: ${ok} ok / ${fail} fail of ${slugs.size} slugs`);
+  // sync cache to prod (non-PII vendor/company previews) so the live panel is fresh
+  try { execFileSync('scp', ['-q', cacheFile, 'my-server:/root/DW-Agents/marketing-command-center/data/vendor-linkedin-cache.json'], { timeout: 30000 }); steps.linkedin.syncedToProd = true; log('LinkedIn cache → prod'); }
+  catch (e) { steps.linkedin.syncedToProd = false; log('scp cache→prod FAILED', e.message); }
+}
+
+// ── 3. Vendor contact sheet (local, PII) ──
+function refreshContacts() {
+  try { execFileSync('node', [path.join(ROOT, 'scripts', 'build-vendor-contact-sheet.mjs')], { timeout: 60000, stdio: 'ignore' }); steps.contacts = { ok: true }; log('contact sheet regenerated'); }
+  catch (e) { steps.contacts = { ok: false, error: e.message }; log('contact sheet FAILED', e.message); }
+}
+
+// ── run ──
+await refreshIG();
+await refreshLinkedIn();
+refreshContacts();
+
+// ── 4. freshness heartbeat (fleet-health vocabulary) ──
+const igOk = steps.ig?.ok, liOk = (steps.linkedin?.ok || 0) > 0;
+const status = (igOk && liOk) ? 'PASS' : (igOk || liOk) ? 'WARN' : 'FAIL';
+fs.writeFileSync(path.join(DATA, 'social-refresh-latest.json'),
+  JSON.stringify({ skill: 'social-refresh', status, verdict: status, ranAt: new Date().toISOString(), steps }, null, 2));
+log('DONE —', status);
← e39578f vendors: paced LinkedIn follows slug-resolver (OG probe + va · back to Marketing Command Center · auto-data-snapshot: 2026-08-31T14:28:57 (1 data files) — pub 793ca1d →