← back to Dw Launches
vendor-ig.js
158 lines
'use strict';
/*
* Vendor-Instagram store + ingest.
*
* Other vendors' Instagram posts are not in dw_unified in any usable form
* (the old `vendor_images` rows are website scrapes that WAF-block on fetch and
* aren't IG). This module owns a self-contained JSON store of REAL vendor IG
* posts whose images are cached to disk, so the Media → Vendor IG tab paints
* reliably and supports Repost / Use.
*
* Store: data/vendor-ig.json
* { handles: [{handle, vendor, name}],
* posts: [{id, handle, vendor, image_url, cached, caption, permalink, posted_at}] }
* Cached image bytes live in data/img-cache (shared with the server img proxy);
* `cached` holds the /api/media/img?... URL that serves the local copy.
*
* Ingest paths (both populate `posts`):
* A) ingestViaGraph() — Meta Graph `business_discovery` (TOS-compliant). Needs
* env META_ACCESS_TOKEN + IG_USER_ID (our IG Business account). FREE.
* B) ingestPermalinks() — for a list of public post URLs, pull the image via
* Instagram's public /embed/ endpoint (no token). FREE, works today, but
* requires knowing the post URLs up front.
*/
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');
const DATA_DIR = path.join(__dirname, 'data');
const STORE = path.join(DATA_DIR, 'vendor-ig.json');
const IMG_CACHE = path.join(DATA_DIR, 'img-cache');
const SEED = path.join(DATA_DIR, 'vendor-ig.seed.json');
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0 Safari/537.36';
function load() {
try { return JSON.parse(fs.readFileSync(STORE, 'utf8')); }
catch {
try { const s = JSON.parse(fs.readFileSync(SEED, 'utf8')); save(s); return s; }
catch { return { handles: [], posts: [] }; }
}
}
function save(s) {
if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true });
fs.writeFileSync(STORE, JSON.stringify(s, null, 2));
}
// normalised rows for the media library (newest first)
function list({ search = '', limit = 60, offset = 0 } = {}) {
const s = load();
let posts = s.posts.slice();
if (search) {
const q = search.toLowerCase();
posts = posts.filter(p =>
(p.caption || '').toLowerCase().includes(q) ||
(p.vendor || '').toLowerCase().includes(q) ||
(p.handle || '').toLowerCase().includes(q));
}
posts.sort((a, b) => new Date(b.posted_at || 0) - new Date(a.posted_at || 0));
return posts.slice(offset, offset + limit).map(p => ({
url: p.cached || p.image_url,
title: p.caption ? p.caption.slice(0, 70) : ('@' + p.handle),
source: 'vendor',
vendor: '@' + p.handle + (p.vendor ? ' · ' + p.vendor : ''),
caption: p.caption || '',
link: p.permalink || ''
}));
}
function count() { return load().posts.length; }
function handles() { return load().handles; }
// --- image caching (writes into the shared img-cache the proxy reads) -------
async function cacheImage(url) {
if (!/^https?:\/\//i.test(url)) return null;
if (!fs.existsSync(IMG_CACHE)) fs.mkdirSync(IMG_CACHE, { recursive: true });
const key = crypto.createHash('sha1').update(url).digest('hex');
const existing = fs.readdirSync(IMG_CACHE).find(f => f.startsWith(key + '.'));
if (!existing) {
let ref; try { ref = new URL(url).origin + '/'; } catch {}
const r = await fetch(url, { headers: { 'User-Agent': UA, ...(ref ? { Referer: ref } : {}), Accept: 'image/*,*/*' } });
if (!r.ok) throw new Error('img ' + r.status);
const ct = r.headers.get('content-type') || 'image/jpeg';
const ext = ct.includes('png') ? 'png' : ct.includes('webp') ? 'webp' : ct.includes('gif') ? 'gif' : 'jpg';
fs.writeFileSync(path.join(IMG_CACHE, key + '.' + ext), Buffer.from(await r.arrayBuffer()));
}
return '/api/media/img?u=' + encodeURIComponent(url);
}
function upsert(store, post) {
const i = store.posts.findIndex(p => p.id === post.id);
if (i >= 0) store.posts[i] = { ...store.posts[i], ...post };
else store.posts.push(post);
}
// --- A) Meta Graph business_discovery --------------------------------------
async function ingestViaGraph({ token = process.env.META_ACCESS_TOKEN, igUserId = process.env.IG_USER_ID, perHandle = 12 } = {}) {
if (!token || !igUserId) throw new Error('META_ACCESS_TOKEN and IG_USER_ID required for Graph ingest');
const store = load();
const out = { ok: [], failed: [], posts: 0 };
for (const h of store.handles) {
try {
const fields = `business_discovery.username(${h.handle}){username,media.limit(${perHandle}){id,media_url,thumbnail_url,permalink,caption,timestamp,media_type}}`;
const u = `https://graph.facebook.com/v21.0/${igUserId}?fields=${encodeURIComponent(fields)}&access_token=${token}`;
const r = await fetch(u);
const j = await r.json();
if (j.error) throw new Error(j.error.message);
const media = (j.business_discovery && j.business_discovery.media && j.business_discovery.media.data) || [];
for (const m of media) {
const img = m.media_type === 'VIDEO' ? (m.thumbnail_url || m.media_url) : m.media_url;
if (!img) continue;
let cached = null; try { cached = await cacheImage(img); } catch {}
upsert(store, {
id: 'ig_' + m.id, handle: h.handle, vendor: h.vendor || h.name,
image_url: img, cached, caption: m.caption || '',
permalink: m.permalink || '', posted_at: m.timestamp || null
});
out.posts++;
}
out.ok.push(h.handle);
} catch (e) { out.failed.push({ handle: h.handle, error: e.message }); }
}
save(store);
return out;
}
// --- B) public /embed/ extraction (no token) -------------------------------
async function ingestPermalinks(urls = []) {
const store = load();
const out = { ok: [], failed: [], posts: 0 };
for (const url of urls) {
try {
const m = url.match(/instagram\.com\/(?:[^/]+\/)?(p|reel|tv)\/([A-Za-z0-9_-]+)/);
if (!m) throw new Error('not an IG post url');
const shortcode = m[2];
const r = await fetch(`https://www.instagram.com/p/${shortcode}/embed/captioned/`, { headers: { 'User-Agent': UA } });
if (!r.ok) throw new Error('embed ' + r.status);
const html = await r.text();
// the embed inlines the display image and (often) the caption
const img = (html.match(/"display_url":"([^"]+)"/) || html.match(/<img[^>]+class="EmbeddedMediaImage"[^>]+src="([^"]+)"/) || [])[1];
const imgUrl = img ? img.replace(/\\u0026/g, '&').replace(/\\\//g, '/') : null;
if (!imgUrl) throw new Error('no image in embed');
const handleMatch = html.match(/instagram\.com\/([A-Za-z0-9_.]+)\/\?utm/) || html.match(/"owner":\{"username":"([^"]+)"/);
const handle = (handleMatch && handleMatch[1]) || 'unknown';
const capMatch = html.match(/class="Caption"[^>]*>([\s\S]*?)<\/div>/);
const caption = capMatch ? capMatch[1].replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 400) : '';
let cached = null; try { cached = await cacheImage(imgUrl); } catch {}
const known = store.handles.find(x => x.handle.toLowerCase() === handle.toLowerCase());
upsert(store, {
id: 'ig_' + shortcode, handle, vendor: known ? (known.vendor || known.name) : '',
image_url: imgUrl, cached, caption, permalink: `https://www.instagram.com/p/${shortcode}/`, posted_at: null
});
out.posts++; out.ok.push(shortcode);
} catch (e) { out.failed.push({ url, error: e.message }); }
}
save(store);
return out;
}
module.exports = { load, save, list, count, handles, cacheImage, ingestViaGraph, ingestPermalinks };