`ingest-browserbase.js`

166 lines
'use strict';
/*
 * Vendor-IG ingest via Browserbase (cloud Chromium, residential IP, defeats the
 * local-IP block). Loads each vendor's public Instagram profile, scrapes recent
 * post thumbnails + permalinks, caches the images locally, and upserts into the
 * vendor-ig store so the Media → Vendor IG tab shows real, cached images.
 *
 * Usage:
 *   node ingest-browserbase.js                 # all seeded handles
 *   node ingest-browserbase.js 3               # first 3 handles (pilot)
 *   node ingest-browserbase.js schumacher1889 thibaut1886   # specific handles
 *
 * PAID: ~$0.01–0.05 per profile (one Browserbase session each). Prints a cost
 * estimate + per-vendor + batch total.
 */
const os = require('os');
const SK = os.homedir() + '/.claude/skills/browserbase';
const Browserbase = require(SK + '/node_modules/@browserbasehq/sdk').default;
const { chromium } = require(SK + '/node_modules/playwright-core');
require(SK + '/node_modules/dotenv').config({ path: SK + '/.env' });
const vendorIg = require('./vendor-ig');

const PER_PROFILE_COST = 0.03; // rough Browserbase session cost, for the ledger
const MAX_POSTS = parseInt(process.env.MAX_POSTS || '12', 10);

function pickHandles(argv) {
  const all = vendorIg.handles();
  if (!argv.length) return all;
  if (argv.length === 1 && /^\d+$/.test(argv[0])) return all.slice(0, parseInt(argv[0], 10));
  const want = new Set(argv.map(s => s.replace(/^@/, '').toLowerCase()));
  return all.filter(h => want.has(h.handle.toLowerCase()));
}

// IG login-walls public profiles; Picuki-family mirrors render them without
// login and are scrapable via the cloud browser (they proxy the real IG image).
// They're clones with the same DOM, so one extractor works across all of them.
// We try each mirror until one yields posts.
const MIRRORS = [
  h => `https://www.picuki.com/profile/${h}`,
  h => `https://www.picnob.com/profile/${h}/`,
  h => `https://www.pixwox.com/profile/${h}/`
];
async function extractPosts(page, max) {
  try { await page.click('text=/Accept|Agree|Consent/i', { timeout: 1500 }); } catch {}
  try { await page.evaluate(() => window.scrollBy(0, 1600)); } catch {}
  await page.waitForTimeout(2200);
  return await page.evaluate((max) => {
    const out = []; const seen = new Set();
    // picuki uses /media/<id>; picnob/pixwox use /post/<id> or /p/<code>
    document.querySelectorAll('a[href*="/media/"], a[href*="/post/"], a[href*="/p/"]').forEach(a => {
      const href = a.href;
      const id = (href.match(/\/(?:media|post|p)\/([A-Za-z0-9_-]+)/) || [])[1];
      if (!id || seen.has(id)) return;
      const img = a.querySelector('img') || a.parentElement.querySelector('img');
      const src = img && (img.getAttribute('src') || (img.getAttribute('srcset') || '').split(' ')[0]);
      if (!src || !/^https?:/.test(src)) return;
      const box = a.closest('.box-photo, .item, li, article, div');
      const desc = box && box.querySelector('.photo-description, .description, .caption');
      seen.add(id);
      out.push({
        shortcode: 'pk' + id, img: src,
        alt: (desc && desc.textContent.trim().slice(0, 300)) || (img && img.getAttribute('alt')) || '',
        permalink: href
      });
    });
    return out.slice(0, max);
  }, max);
}
// CAPTURE=shot → grab each post thumbnail as a rendered element-screenshot
// inside the cloud browser (bypasses mirrors' signed/expiring image URLs).
// Returns posts with a `shot` Buffer instead of relying on a fetchable img url.
async function captureShots(page, max) {
  try { await page.click('text=/Accept|Agree|Consent/i', { timeout: 1500 }); } catch {}
  try { await page.evaluate(() => window.scrollBy(0, 1600)); } catch {}
  await page.waitForTimeout(2500);
  const anchors = await page.$$('a[href*="/media/"], a[href*="/post/"], a[href*="/p/"]');
  const out = []; const seen = new Set();
  for (const a of anchors) {
    if (out.length >= max) break;
    try {
      const href = await a.getAttribute('href');
      const id = (href && href.match(/\/(?:media|post|p)\/([A-Za-z0-9_-]+)/) || [])[1];
      if (!id || seen.has(id)) continue;
      const img = await a.$('img');
      if (!img) continue;
      const box = await img.boundingBox();
      if (!box || box.width < 80) continue; // skip icons/avatars
      const alt = (await img.getAttribute('alt')) || '';
      const shot = await img.screenshot({ type: 'jpeg', quality: 88 });
      seen.add(id);
      out.push({ shortcode: 'pk' + id, alt: alt.slice(0, 300), permalink: href, shot });
    } catch {}
  }
  return out;
}
async function scrapeProfile(page, handle) {
  let lastErr = 'no mirror yielded posts';
  const useShot = process.env.CAPTURE === 'shot';
  for (const m of MIRRORS) {
    try {
      const resp = await page.goto(m(handle), { waitUntil: 'domcontentloaded', timeout: 40000 });
      if (resp && resp.status() >= 400) { lastErr = 'mirror ' + resp.status(); continue; }
      await page.waitForTimeout(2800);
      if (/no posts|user has no|not found/i.test(await page.evaluate(() => document.body.innerText.slice(0, 400)))) {
        lastErr = 'empty profile'; continue;
      }
      const posts = useShot ? await captureShots(page, MAX_POSTS) : await extractPosts(page, MAX_POSTS);
      if (posts.length) return posts;
      lastErr = '0 posts extracted';
    } catch (e) { lastErr = e.message.slice(0, 50); }
  }
  throw new Error(lastErr);
}

(async () => {
  const handles = pickHandles(process.argv.slice(2));
  if (!handles.length) { console.error('no handles matched'); process.exit(1); }
  const estLow = (handles.length * 0.01).toFixed(2), estHigh = (handles.length * 0.05).toFixed(2);
  console.log(`[browserbase] ${handles.length} profile(s) — est cost $${estLow}–$${estHigh}`);

  const bb = new Browserbase({ apiKey: process.env.BROWSERBASE_API_KEY });
  const summary = [];
  let spend = 0;

  for (const h of handles) {
    let session, browser, posts = 0, imgs = 0, err = null;
    try {
      session = await bb.sessions.create({ projectId: process.env.BROWSERBASE_PROJECT_ID });
      browser = await chromium.connectOverCDP(session.connectUrl);
      const ctx = browser.contexts()[0];
      const page = ctx.pages()[0] || await ctx.newPage();
      const found = await scrapeProfile(page, h.handle);
      posts = found.length;
      const store = vendorIg.load();
      const fs = require('fs'), path = require('path');
      const CACHE = path.join(__dirname, 'data', 'img-cache');
      if (!fs.existsSync(CACHE)) fs.mkdirSync(CACHE, { recursive: true });
      for (const f of found) {
        let cached = null, image_url = f.img || f.permalink;
        if (f.shot) {
          // in-session screenshot → save locally, serve via /api/vendor-ig/img/:id
          try { fs.writeFileSync(path.join(CACHE, 'vig_' + f.shortcode + '.jpg'), f.shot); cached = '/api/vendor-ig/img/' + f.shortcode; imgs++; } catch {}
        } else {
          try { cached = await vendorIg.cacheImage(f.img); imgs++; } catch {}
        }
        const i = store.posts.findIndex(p => p.id === 'ig_' + f.shortcode);
        const rec = {
          id: 'ig_' + f.shortcode, handle: h.handle, vendor: h.vendor || h.name,
          image_url, cached, caption: f.alt || '',
          permalink: f.permalink || '', posted_at: null
        };
        if (i >= 0) store.posts[i] = { ...store.posts[i], ...rec }; else store.posts.push(rec);
      }
      vendorIg.save(store);
    } catch (e) { err = e.message; }
    finally { try { await browser?.close(); } catch {} }
    spend += PER_PROFILE_COST;
    summary.push({ handle: h.handle, posts, imgs, err });
    console.log(`  @${h.handle.padEnd(20)} posts:${posts} cached:${imgs}${err ? ' ERR:' + err.slice(0, 60) : ''}  [batch $${spend.toFixed(2)}]`);
  }

  const okV = summary.filter(s => s.imgs > 0).length;
  const totalImgs = summary.reduce((a, s) => a + s.imgs, 0);
  console.log(`\n[done] ${okV}/${handles.length} vendors yielded images · ${totalImgs} images cached · ~$${spend.toFixed(2)} spent · store now ${vendorIg.count()} posts`);
})().catch(e => { console.error('FATAL', e.message); process.exit(1); });