← back to Marketing Command Center
modules/vendors/index.js
321 lines
// Vendor IG Reporting — directory of DW's + all vendor Instagram accounts, for
// competitive/social reporting. Reads data/vendor-instagram.json (curated; refresh
// via research). Now also fetches each account's LAST 10 POSTS via Instagram
// Business Discovery (DW's own IG business account can read any public IG business
// account's recent media). Posts are cached to data/vendor-posts-cache.json; the
// grid renders from cache, and POST /posts/refresh re-crawls. Read-only re: writes.
const fs = require('fs');
const path = require('path');
const GRAPH = 'https://graph.facebook.com/v23.0';
const DATA = path.join(__dirname, '..', '..', 'data', 'vendor-instagram.json');
const CACHE = path.join(__dirname, '..', '..', 'data', 'vendor-posts-cache.json');
// LinkedIn vendor→company mapping + public-OG post cache (reference/attribution-
// amplify only — NO LinkedIn API, NO auth, NO scraping-around-login; a read-only
// Open-Graph GET of the PUBLIC company/post page, exactly like the linkedin
// module's fetchOg helper).
const LI_DATA = path.join(__dirname, '..', '..', 'data', 'vendor-linkedin.json');
const LI_CACHE = path.join(__dirname, '..', '..', 'data', 'vendor-linkedin-cache.json');
function load() { try { return JSON.parse(fs.readFileSync(DATA, 'utf8')); } catch { return []; } }
function loadCache() { try { return JSON.parse(fs.readFileSync(CACHE, 'utf8')); } catch { return {}; } }
function saveCache(c) { fs.writeFileSync(CACHE, JSON.stringify(c, null, 2)); }
const LI_FOLLOWS = path.join(__dirname, '..', '..', 'data', 'vendor-linkedin-follows.json');
function loadLi() {
let base; try { base = JSON.parse(fs.readFileSync(LI_DATA, 'utf8')); } catch { base = { accounts: [] }; }
base.accounts = base.accounts || [];
// Merge the "firms Steve follows" export (import-linkedin-follows.mjs) if present.
// Curated vendor map wins on a slug collision; new followed firms are appended.
try {
const foll = JSON.parse(fs.readFileSync(LI_FOLLOWS, 'utf8'));
const have = new Set(base.accounts.filter(a => a.slug).map(a => a.slug.toLowerCase()));
for (const a of (foll.accounts || [])) {
if (a.slug && !have.has(a.slug.toLowerCase())) { base.accounts.push(a); have.add(a.slug.toLowerCase()); }
}
} catch { /* no follows file yet — fine */ }
return base;
}
function loadLiCache() { try { return JSON.parse(fs.readFileSync(LI_CACHE, 'utf8')); } catch { return {}; } }
function saveLiCache(c) { try { fs.mkdirSync(path.dirname(LI_CACHE), { recursive: true }); fs.writeFileSync(LI_CACHE, JSON.stringify(c, null, 2)); } catch { /* non-fatal */ } }
function envFrom(file, k) { try { return (fs.readFileSync(file, 'utf8').match(new RegExp('^' + k + '=(.+)$', 'm')) || [])[1]; } catch { return null; } }
// Resolve a usable Meta token — prefer a never-expiring PAGE token. Checks
// META_ACCESS_TOKEN then IG_ACCESS_TOKEN, in MCC .env then the secrets master.
function metaToken() {
const files = [path.join(__dirname, '..', '..', '.env'), path.join(require('os').homedir(), 'Projects/secrets-manager/.env')];
for (const k of ['IG_ACCESS_TOKEN', 'META_ACCESS_TOKEN']) for (const f of files) {
const v = envFrom(f, k);
if (v && !/TOKEN\s*$/i.test(v) && v.startsWith('EAA')) return v;
}
return null;
}
function followersToNum(f) {
if (!f) return 0;
const m = String(f).trim().match(/^([\d.]+)\s*([kKmM]?)/);
if (!m) return 0;
const n = parseFloat(m[1]); const u = m[2].toLowerCase();
return Math.round(n * (u === 'm' ? 1e6 : u === 'k' ? 1e3 : 1));
}
const handleOf = r => String(r.handle || '').replace(/^@/, '').trim();
const sleep = ms => new Promise(r => setTimeout(r, ms));
// Find a DW-owned IG business account id to use as the business_discovery node.
let _igNode = null;
async function discoveringIg(token) {
if (_igNode) return _igNode;
// page-token path: me == the Page itself
let r = await fetch(`${GRAPH}/me?fields=name,instagram_business_account{id,username}&access_token=${encodeURIComponent(token)}`);
let j = await r.json();
if (!j.error && j.instagram_business_account) {
_igNode = { id: j.instagram_business_account.id, username: j.instagram_business_account.username };
return _igNode;
}
// user-token path: list Pages, find one with a linked IG account
r = await fetch(`${GRAPH}/me/accounts?fields=name,instagram_business_account{id,username}&limit=100&access_token=${encodeURIComponent(token)}`);
j = await r.json();
if (j.error) throw new Error(j.error.message);
const p = (j.data || []).find(p => p.instagram_business_account);
if (!p) throw new Error('no IG business account linked to any Page');
_igNode = { id: p.instagram_business_account.id, username: p.instagram_business_account.username };
return _igNode;
}
async function fetchPosts(igId, token, username) {
const fields = `business_discovery.username(${username}){username,followers_count,media_count,media.limit(10){id,caption,media_type,media_url,thumbnail_url,permalink,timestamp,like_count,comments_count}}`;
const r = await fetch(`${GRAPH}/${igId}?fields=${encodeURIComponent(fields)}&access_token=${encodeURIComponent(token)}`);
const j = await r.json();
if (j.error) throw new Error(j.error.message);
const bd = j.business_discovery || {};
return {
followers_count: bd.followers_count ?? null,
media_count: bd.media_count ?? null,
posts: (bd.media?.data || []).map(m => ({
id: m.id, caption: (m.caption || '').slice(0, 280), media_type: m.media_type,
image: m.media_type === 'VIDEO' ? (m.thumbnail_url || m.media_url) : m.media_url,
permalink: m.permalink, timestamp: m.timestamp,
likes: m.like_count ?? null, comments: m.comments_count ?? null,
})),
};
}
// ── LinkedIn public Open-Graph harvest (reference/attribution-amplify) ────────
// LinkedIn has NO public feed-read API and its User Agreement bans scraping-
// around-login, so this is a best-effort READ-ONLY Open-Graph GET of the PUBLIC
// company page (linkedin.com/company/<slug>) — same technique as the linkedin
// module's fetchOg. It returns the company page's og:title/og:image/og:description
// (a thumbnail + text + the permalink). It NEVER logs in, never hits an API,
// never downloads the image into DW's assets — the amplify UX copies the text and
// shares a link WITH attribution back to the vendor.
const LI_UA = 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)';
async function fetchLiOg(url) {
// Defense-in-depth (the slug is always from our own JSON, never user input, so
// this is belt-and-suspenders): only ever fetch a linkedin.com company page.
try { const h = new URL(url).hostname; if (!/(^|\.)linkedin\.com$/i.test(h)) return { ok: false, error: 'refusing non-linkedin host' }; } catch { return { ok: false, error: 'bad url' }; }
const ctrl = new AbortController();
const timer = setTimeout(() => ctrl.abort(), 8000);
try {
// redirect:'manual' so a crafted redirect can't bounce this fetch off-host.
const r = await fetch(url, { headers: { 'User-Agent': LI_UA, 'Accept-Language': 'en-US,en;q=0.9' }, redirect: 'manual', signal: ctrl.signal });
const status = r.status;
if (status >= 300 && status < 400) return { ok: false, status, error: 'redirected (auth wall)' };
if (!r.ok) return { ok: false, status, error: 'HTTP ' + status + (status === 999 || status === 429 ? ' (LinkedIn rate-limited / auth-walled this fetch)' : '') };
if (!/text\/html/i.test(r.headers.get('content-type') || '')) return { ok: false, status, error: 'non-html response' };
const html = (await r.text()).slice(0, 400000);
// Decode HTML entities (LinkedIn double-encodes &amp; → run &→& twice).
const dec = s => String(s || '').replace(/&/g, '&').replace(/&/g, '&').replace(/'/g, "'").replace(/'/g, "'").replace(/"/g, '"').replace(/</g, '<').replace(/>/g, '>');
const og = k => { const m = html.match(new RegExp('<meta[^>]+(?:property|name)=["\\\']og:' + k + '["\\\'][^>]+content=["\\\']([^"\\\']+)', 'i')); return m ? dec(m[1]) : ''; };
return { ok: true, status, title: og('title'), thumb: og('image'), description: og('description') };
} catch (e) {
return { ok: false, error: /abort/i.test(e.message) ? 'timeout' : e.message };
} finally { clearTimeout(timer); }
}
// Reference/attribution-amplify kit for a LinkedIn company: copyable text that
// CREDITS the vendor + links back to their public page, plus a DW-visit CTA. No
// image is ever pulled into DW assets — attribution + link-share only.
// DW standing rule — "Wallpaper" is banned in DW's own output; "Wallcovering(s)".
// The amplify CAPTION is DW copy (even though it quotes the vendor), so de-wallpaper it.
const deWallpaper = s => String(s || '').replace(/\bwallpapers\b/gi, 'wallcoverings').replace(/\bwallpaper\b/gi, 'wallcovering');
function liAmplifyKit(brand, companyUrl, og) {
const DWUTM = 'https://designerwallcoverings.com/?utm_source=linkedin&utm_medium=social&utm_campaign=vendor-amplify';
const snippet = deWallpaper(String(og && og.description || og && og.title || '').replace(/\s+/g, ' ').trim().slice(0, 220));
const text = `${brand}${snippet ? ' — ' + snippet : ''}\n\n🔗 via ${brand} on LinkedIn: ${companyUrl}\n🛍️ Explore the line at Designer Wallcoverings: ${DWUTM}\n\n#DesignerWallcoverings #wallcoverings #interiordesign`;
return { text, dwLink: DWUTM };
}
module.exports = {
id: 'vendors',
title: 'Vendor IG Reporting',
icon: '📷',
mount(router) {
router.get('/accounts', (_req, res) => {
const cache = loadCache();
const rows = load().map(r => {
const h = handleOf(r);
const c = cache[h] || null;
return {
...r,
followersNum: followersToNum(r.followersVerified || r.followers),
followersDisplay: r.followersVerified || r.followers,
verifiedLive: r.live === true && !!r.followersVerified,
hasIG: r.handle && r.handle !== 'none found',
posts: c?.posts || [],
postsFetchedAt: c?.fetchedAt || null,
postsError: c?.error || null,
};
});
const withIG = rows.filter(r => r.hasIG);
const reach = withIG.reduce((s, r) => s + r.followersNum, 0);
const withPosts = rows.filter(r => r.posts && r.posts.length).length;
res.json({
accounts: rows,
stats: {
total: rows.length, withIG: withIG.length, missing: rows.length - withIG.length,
totalReach: reach, withPosts,
postsFetchedAt: Object.values(cache).map(c => c.fetchedAt).filter(Boolean).sort().pop() || null,
dw: rows.find(r => r.vendorCode === 'dw') || null,
},
});
});
// Re-crawl last-10 posts for all (or ?handle=) vendors via business_discovery.
router.post('/posts/refresh', async (req, res) => {
const token = metaToken();
if (!token) return res.json({ ok: false, error: 'No usable Meta token (META_ACCESS_TOKEN / IG_ACCESS_TOKEN)' });
let ig;
try { ig = await discoveringIg(token); }
catch (e) { return res.json({ ok: false, error: 'token/IG check failed: ' + e.message + (/expired|session/i.test(e.message) ? ' — paste a fresh long-lived META_ACCESS_TOKEN' : '') }); }
const only = (req.query.handle || '').replace(/^@/, '').trim();
const cache = loadCache();
const targets = load().filter(r => { const h = handleOf(r); return h && r.handle !== 'none found' && (!only || h === only); });
let ok = 0, failed = 0;
for (const r of targets) {
const h = handleOf(r);
try {
const data = await fetchPosts(ig.id, token, h);
cache[h] = { ...data, fetchedAt: new Date().toISOString(), error: null };
ok++;
} catch (e) {
cache[h] = { ...(cache[h] || {}), error: e.message, fetchedAt: new Date().toISOString() };
failed++;
}
saveCache(cache);
await sleep(600); // be gentle on Graph rate limits
}
res.json({ ok: true, discoveringIg: '@' + ig.username, refreshed: ok, failed, total: targets.length });
});
// Refresh ONE account's recent media and RETURN its fresh posts. This is the
// "Make it ours" refresh-on-click path (TK-10843): the branded-card compositor
// sources p.image, which is a SIGNED IG CDN url that expires (~a day) — a cached
// post that's days old returns 403 → /api/img-proxy 502s → the card can't
// compose. So before compositing, the client fetches a FRESH image url for that
// specific post via a targeted, single-handle refresh.
//
// Reuses the EXACT bulk-refresh code path (metaToken → discoveringIg →
// fetchPosts), scoped to one handle, and writes the SAME cache the bulk refresh
// writes (data/vendor-posts-cache.json) so the two stay consistent. It returns
// the account's fresh posts (id / permalink / image / timestamp / caption) so the
// client can match the clicked post to its refreshed image url.
//
// COST: Business Discovery is free within Meta rate limits (no metered call).
// GRACEFUL DEGRADE: a dead/expired Meta token (see the meta-token-canary
// dependency), a rate-limit, or an unknown handle returns { ok:false, error } —
// the client shows a clear inline message and does NOT hard-error.
router.post('/posts/refresh-one', async (req, res) => {
const handle = String((req.body && req.body.handle) || (req.query && req.query.handle) || '')
.replace(/^@/, '').trim().slice(0, 80);
if (!handle) return res.json({ ok: false, error: 'no handle' });
// Only refresh a handle that's actually in the vendor roster (don't let an
// arbitrary username be used to probe Business Discovery through our token).
const known = load().some(r => handleOf(r).toLowerCase() === handle.toLowerCase() && r.handle !== 'none found');
if (!known) return res.json({ ok: false, error: 'unknown handle' });
const token = metaToken();
if (!token) return res.json({ ok: false, error: 'No usable Meta token (META_ACCESS_TOKEN / IG_ACCESS_TOKEN) — check the meta-token canary' });
let ig;
try { ig = await discoveringIg(token); }
catch (e) { return res.json({ ok: false, error: 'token/IG check failed: ' + e.message + (/expired|session/i.test(e.message) ? ' — paste a fresh long-lived META_ACCESS_TOKEN' : '') }); }
const cache = loadCache();
try {
const data = await fetchPosts(ig.id, token, handle);
cache[handle] = { ...data, fetchedAt: new Date().toISOString(), error: null };
saveCache(cache);
// Business Discovery is not a metered call — no cost to report.
res.json({ ok: true, handle, discoveringIg: '@' + ig.username, fetchedAt: cache[handle].fetchedAt, posts: data.posts || [] });
} catch (e) {
cache[handle] = { ...(cache[handle] || {}), error: e.message, fetchedAt: new Date().toISOString() };
saveCache(cache);
res.json({ ok: false, handle, error: e.message + (/rate|limit|#4|#17/i.test(e.message) ? ' — rate-limited, try again shortly' : '') });
}
});
// ── LinkedIn (reference/attribution-amplify) ─────────────────────────────
// GET the vendor→LinkedIn-company roster merged with the cached OG pull +
// a ready-to-copy attribution-amplify kit per vendor. READ-ONLY; no API.
router.get('/linkedin/accounts', (_req, res) => {
const cfg = loadLi();
const cache = loadLiCache();
const accounts = (cfg.accounts || []).map(a => {
const companyUrl = a.slug ? `https://www.linkedin.com/company/${a.slug}/` : null;
const c = a.slug ? (cache[a.slug] || null) : null;
const og = c && c.og ? c.og : null;
return {
vendorCode: a.vendorCode, brand: a.brand, slug: a.slug || null,
verified: a.verified === true, note: a.note || '',
hasLinkedIn: !!a.slug, companyUrl,
og, fetchedAt: c ? c.fetchedAt : null, error: c ? c.error : null,
amplify: (companyUrl && og) ? liAmplifyKit(a.brand, companyUrl, og) : null,
};
});
const withLi = accounts.filter(a => a.hasLinkedIn);
res.json({
accounts,
stats: {
total: accounts.length,
withLinkedIn: withLi.length,
verified: withLi.filter(a => a.verified).length,
unverified: withLi.filter(a => !a.verified).length,
missing: accounts.length - withLi.length,
harvested: accounts.filter(a => a.og).length,
harvestedAt: Object.values(cache).map(c => c.fetchedAt).filter(Boolean).sort().pop() || null,
},
});
});
// POST /linkedin/harvest — pull the PUBLIC OG (thumbnail + text) for every
// mapped vendor company page (or ?vendorCode= for one). Read-only GETs; no
// login, no API. Rate-gentle. Honest per-vendor error surfacing (LinkedIn may
// auth-wall / rate-limit a given fetch — the embed/link still works).
router.post('/linkedin/harvest', async (req, res) => {
const only = String((req.body && req.body.vendorCode) || (req.query && req.query.vendorCode) || '').trim();
const cfg = loadLi();
const cache = loadLiCache();
const targets = (cfg.accounts || []).filter(a => a.slug && (!only || a.vendorCode === only));
let ok = 0, failed = 0;
for (const a of targets) {
const companyUrl = `https://www.linkedin.com/company/${a.slug}/`;
const og = await fetchLiOg(companyUrl);
if (og.ok) { cache[a.slug] = { og: { title: og.title, thumb: og.thumb, description: og.description }, status: og.status, fetchedAt: new Date().toISOString(), error: null }; ok++; }
else { cache[a.slug] = { ...(cache[a.slug] || {}), status: og.status || null, fetchedAt: new Date().toISOString(), error: og.error || 'fetch failed' }; failed++; }
saveLiCache(cache);
await sleep(800); // be gentle — LinkedIn rate-limits aggressive OG pulls
}
res.json({ ok: true, harvested: ok, failed, total: targets.length });
});
// GET /contacts — "who I know inside each DW vendor" (LinkedIn connections ×
// vendor_registry, built by scripts/build-vendor-contact-sheet.mjs). INTERNAL/PII;
// this panel is Basic-Auth gated. Reference only — never a public surface.
const LI_CONTACTS_JSON = path.join(__dirname, '..', '..', 'data', 'vendor-contacts.json');
const LI_CONTACTS_HTML = path.join(__dirname, '..', '..', 'data', 'vendor-contacts.html');
router.get('/contacts', (_req, res) => {
try { res.json(JSON.parse(fs.readFileSync(LI_CONTACTS_JSON, 'utf8'))); }
catch { res.json({ vendors: 0, totalContacts: 0, rows: [], note: 'Run scripts/build-vendor-contact-sheet.mjs first.' }); }
});
router.get('/contacts/sheet', (_req, res) => {
try { res.type('html').send(fs.readFileSync(LI_CONTACTS_HTML, 'utf8')); }
catch { res.type('html').send('<p style="font:16px sans-serif;margin:40px">No contact sheet yet — run <code>scripts/build-vendor-contact-sheet.mjs</code>.</p>'); }
});
},
};