`scripts/resolve-follows.mjs`

66 lines
// Resolve the 398 followed-firm slugs (data/vendor-linkedin-follows.json) by probing
// LinkedIn's PUBLIC company page Open-Graph — naive slug + a few variant fallbacks —
// GENTLY (sequential, delay, early-stop on repeated rate-walls). Writes resolved slugs
// back so re-runs skip them. Reference/attribution-amplify only; read-only public GETs.
//   node scripts/resolve-follows.mjs
import fs from 'node:fs';
import path from 'node:path';

const F = path.join(path.dirname(new URL(import.meta.url).pathname), '..', 'data', 'vendor-linkedin-follows.json');
const doc = JSON.parse(fs.readFileSync(F, 'utf8'));
const sleep = ms => new Promise(r => setTimeout(r, ms));

const slugify = s => s.toLowerCase().trim().replace(/&/g, ' and ').replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '');
// ordered candidate slugs for a company name
function candidates(brand) {
  const base = brand.replace(/\([^)]*\)/g, '').trim();       // drop parentheticals
  const noSuffix = base.replace(/\b(inc|llc|ltd|co|corp|company|plc|group|gmbh)\b\.?/gi, '').trim();
  const set = new Set([
    slugify(brand), slugify(base), slugify(noSuffix),
    slugify(noSuffix.replace(/\band\b/gi, '')),              // drop 'and'
    slugify(noSuffix.split(/\s+/)[0] || ''),                 // first token
  ]);
  return [...set].filter(s => s && s.length >= 3);
}

async function ogTitle(slug) {
  const url = `https://www.linkedin.com/company/${slug}/`;
  try {
    const r = await fetch(url, { redirect: 'manual', headers: {
      'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36',
      'accept': 'text/html' } });
    if (r.status === 999 || r.status === 429) return { wall: true, status: r.status };
    if (r.status >= 300 && r.status < 400) return { status: r.status };   // slug redirected (usually not a real page)
    if (!r.ok) return { status: r.status };
    const html = await r.text();
    const m = html.match(/<meta property="og:title" content="([^"]+)"/i);
    return { status: r.status, title: m ? m[1].replace(/ \| LinkedIn$/, '').trim() : null };
  } catch (e) { return { error: e.message }; }
}

const norm = s => (s || '').toLowerCase().replace(/[^a-z0-9]/g, '');
let resolved = 0, unresolved = 0, walls = 0, consecWall = 0, tried = 0;
const acc = doc.accounts;
for (let i = 0; i < acc.length; i++) {
  const a = acc[i];
  if (a.verified && a.resolvedAt) continue;                 // already resolved on a prior run
  let hit = null;
  for (const slug of candidates(a.brand)) {
    tried++;
    const og = await ogTitle(slug);
    await sleep(1500);                                       // gentle
    if (og.wall) { walls++; consecWall++; if (consecWall >= 8) { console.log(`RATE-WALLED after ${i} firms — stopping early, ${resolved} resolved so far.`); i = acc.length; break; } continue; }
    consecWall = 0;
    if (og.title) {                                         // plausibility: brand tokens overlap the og:title
      const bn = norm(a.brand), tn = norm(og.title);
      if (bn && tn && (tn.includes(bn.slice(0, 6)) || bn.includes(tn.slice(0, 6)))) { hit = { slug, title: og.title }; break; }
    }
  }
  if (hit) { a.slug = hit.slug; a.verified = true; a.ogTitle = hit.title; a.resolvedAt = 'set'; resolved++; }
  else { a.verified = false; unresolved++; }
  if (i % 25 === 0) fs.writeFileSync(F, JSON.stringify(doc, null, 2));   // checkpoint
}
doc.resolvedSummary = { resolved, unresolved, walls, tried };
fs.writeFileSync(F, JSON.stringify(doc, null, 2));
console.log(`DONE: resolved ${resolved} / unresolved ${unresolved} / rate-walls ${walls} / probes ${tried}`);