`scripts/linkedin-feed-harvest.mjs`

150 lines
#!/usr/bin/env node
// linkedin-feed-harvest.mjs — LOCAL research aid for the Marketing Command Center.
//
// LinkedIn has NO feed-read API (the Community Management API the MCC uses only
// PUBLISHES + reads DW's own org posts). Steve authorized (2026-08-12, TK-10504)
// harvesting HIS OWN home feed through openclaw's real logged-in Chrome so the
// "Network feed" panel can surface video posts worth resharing/repurposing.
//
// This is a MANUAL, human-triggered tool — it is NEVER an always-on auto-poster.
// It only READS the feed and writes data/linkedin-feed.json. Every outward action
// (reshare / download) stays gated in the panel. Scraping bypasses bot-detection,
// not gates: dw-legal-compliance + the reshare/publish confirm gates still apply.
//
// ⚠ STATUS (2026-08-12, TK-10504): EXPERIMENTAL SCAFFOLD — currently returns 0
// posts against LinkedIn's live feed. Empirically probed: LinkedIn serves the
// CDP/automation-controlled Chrome a stripped feed with (1) hashed/randomized
// post class names (.feed-shared-update-v2 / [data-urn] match nothing), (2) no
// urn:li:activity in any DOM attribute, (3) blob:/MSE video (data-sources gone,
// nothing downloadable in the DOM), and (4) no fetch/XHR feed calls on scroll.
// The next path (Voyager API via session cookie + CSRF) is forbidden token-
// harvesting + TOS-violating and is intentionally NOT taken. Kept as a scaffold
// in case a viable, compliant source appears. The selectors below are the
// last-known-good shape and will need reworking if LinkedIn re-exposes the feed.
//
// Usage:
//   node scripts/linkedin-feed-harvest.mjs                 # video posts, 8 scrolls
//   node scripts/linkedin-feed-harvest.mjs --scrolls=12    # scroll deeper
//   node scripts/linkedin-feed-harvest.mjs --all           # keep non-video posts too
//
// Cost: $0 (openclaw real browser + local). Requires openclaw enabled + Chrome
// already logged into LinkedIn on the `openclaw` profile.
import { execSync } from 'child_process';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const OUT = path.join(__dirname, '..', 'data', 'linkedin-feed.json');
const kv = Object.fromEntries(process.argv.slice(2).filter(a => a.includes('=')).map(a => a.slice(2).split('=')));
const FLAGS = new Set(process.argv.slice(2).filter(a => !a.includes('=')).map(a => a.replace(/^--/, '')));
const SCROLLS = Math.max(1, Math.min(40, Number(kv.scrolls) || 8));
const VIDEO_ONLY = !FLAGS.has('all');
const sleep = ms => new Promise(r => setTimeout(r, ms));

function oc(args) {
  return execSync(`openclaw browser ${args}`, { encoding: 'utf8', timeout: 60000, stdio: ['ignore', 'pipe', 'ignore'] });
}
// openclaw evaluate returns a JSON-encoded string; a fn that returns
// JSON.stringify(x) comes back double-encoded → parse up to twice.
function unwrap(s) {
  s = (s || '').trim().split('\n').filter(Boolean).pop() || 'null';
  try { let v = JSON.parse(s); if (typeof v === 'string') { try { v = JSON.parse(v); } catch { /* plain string */ } } return v; }
  catch { return null; }
}

// Extraction runs INSIDE the page. Written with string concat + .includes()/
// .endsWith() so it carries no `$` or backticks (shell-hazard chars when inlined
// into the openclaw --fn argument under double quotes).
const EXTRACT_FN = "() => {" +
  "var out=[];" +
  "var nodes=document.querySelectorAll('[data-urn]');" +
  "for(var i=0;i<nodes.length;i++){var n=nodes[i];" +
    "var urn=n.getAttribute('data-urn')||'';" +
    "if(urn.indexOf('urn:li:activity:')!==0)continue;" +
    "var authEl=n.querySelector('.update-components-actor__title, .update-components-actor__name, span.feed-shared-actor__name');" +
    "var author=authEl?authEl.innerText.trim().split('\\n')[0]:'';" +
    "var subEl=n.querySelector('.update-components-actor__description');" +
    "var sub=subEl?subEl.innerText.trim().split('\\n')[0]:'';" +
    "var txtEl=n.querySelector('.update-components-update-v2__commentary, .feed-shared-update-v2__description, .update-components-text');" +
    "var text=txtEl?txtEl.innerText.trim():'';" +
    // gather progressive mp4 sources from any descendant data-sources blob
    "var srcs=[];var ds=n.querySelectorAll('[data-sources]');" +
    "for(var j=0;j<ds.length;j++){try{var arr=JSON.parse(ds[j].getAttribute('data-sources'));" +
      "for(var k=0;k<arr.length;k++){var s=arr[k]||{};var u=s.src||'';" +
        "if(u.indexOf('licdn')!==-1&&(u.indexOf('/mp4-')!==-1||u.endsWith('.mp4'))){" +
          "srcs.push({src:u,bitrate:Number(s['data-bitrate']||s.bitrate||0)});}}}catch(e){}}" +
    // fallback: a bare <video src>
    "if(!srcs.length){var v=n.querySelector('video');if(v&&v.src&&v.src.indexOf('licdn')!==-1&&v.src.indexOf('blob:')!==0){srcs.push({src:v.src,bitrate:0});}}" +
    "srcs.sort(function(a,b){return b.bitrate-a.bitrate;});" +
    "var vEl=n.querySelector('video');" +
    "var poster=vEl?vEl.getAttribute('poster'):'';" +
    "if(!poster){var img=n.querySelector('.update-components-linkedin-video img, .ivm-image-view-model img');poster=img?img.src:'';}" +
    "out.push({urn:urn,author:author,sub:sub,text:text.slice(0,600),isVideo:srcs.length>0," +
      "video:srcs.length?{sources:srcs,best:srcs[0].src,poster:poster}:null," +
      "permalink:'https://www.linkedin.com/feed/update/'+urn+'/'});}" +
  // de-dup by urn, preserving order
  "var seen={};var uniq=[];for(var m=0;m<out.length;m++){if(!seen[out[m].urn]){seen[out[m].urn]=1;uniq.push(out[m]);}}" +
  "return JSON.stringify(uniq);}";

function ensureOpenclaw() {
  let status;
  try { status = oc('status'); } catch (e) { throw new Error('openclaw not reachable — is the CLI installed + Chrome available? (' + e.message + ')'); }
  if (!/enabled/i.test(status) || /enabled\s*[:=]\s*false/i.test(status)) {
    throw new Error('openclaw browser is not enabled. Run: openclaw browser status  (expect enabled:true)');
  }
}

async function main() {
  ensureOpenclaw();
  console.log('Opening LinkedIn feed in openclaw real Chrome…');
  const openOut = oc('open "https://www.linkedin.com/feed/" --timeout 40000');
  const tab = (openOut.match(/id:\s*([A-F0-9]+)/i) || [])[1];
  if (!tab) throw new Error('could not obtain a tab id from openclaw open');

  await sleep(3500);
  // login guard — an authwall/login redirect means the openclaw Chrome profile
  // isn't signed into LinkedIn. Don't hammer; tell Steve to log in once.
  const href = unwrap(oc(`evaluate --target-id ${tab} --fn ${JSON.stringify('() => location.href')}`)) || '';
  if (/\/(login|authwall|uas\/login|checkpoint)/i.test(String(href))) {
    try { oc(`close --target-id ${tab}`); } catch {}
    throw new Error('LinkedIn is not logged in on the openclaw Chrome profile (landed on ' + href + '). Log into linkedin.com once in that Chrome, then re-run.');
  }

  // scroll to lazy-load posts
  let count = 0;
  for (let i = 0; i < SCROLLS; i++) {
    try { count = unwrap(oc(`evaluate --target-id ${tab} --fn ${JSON.stringify('() => { window.scrollBy(0, document.body.scrollHeight); return document.querySelectorAll("[data-urn]").length; }')}`)) || count; } catch {}
    process.stdout.write(`  scroll ${i + 1}/${SCROLLS} — ${count} update containers loaded\r`);
    await sleep(1600);
  }
  console.log('');

  let posts = [];
  for (let i = 0; i < 4 && !posts.length; i++) {
    await sleep(1200);
    const v = unwrap(oc(`evaluate --target-id ${tab} --fn ${JSON.stringify(EXTRACT_FN)}`));
    if (Array.isArray(v)) posts = v;
  }
  try { oc(`close --target-id ${tab}`); } catch {}

  const videoPosts = posts.filter(p => p.isVideo);
  const kept = VIDEO_ONLY ? videoPosts : posts;
  const payload = {
    harvestedAt: new Date().toISOString(),
    source: 'openclaw-real-chrome',
    scrolls: SCROLLS,
    videoOnly: VIDEO_ONLY,
    total: posts.length,
    videos: videoPosts.length,
    count: kept.length,
    posts: kept,
  };
  fs.mkdirSync(path.dirname(OUT), { recursive: true });
  fs.writeFileSync(OUT, JSON.stringify(payload, null, 2));
  console.log(`Harvested ${posts.length} posts (${videoPosts.length} with video). Wrote ${kept.length} → ${path.relative(process.cwd(), OUT)}`);
  console.log('$0 (openclaw real browser + local). Note: licdn video URLs are signed + expire — download soon.');
}

main().catch(e => { console.error('linkedin-feed-harvest error:', e.message); process.exitCode = 1; });