← back to Dw Launches

Vendor-IG: multi-mirror scraper (picuki/picnob/pixwox); keep only cacheable-image posts (picnob URLs ephemeral)

18fb32f20fa6aa19f46ec0aa349a872329aec32f · 2026-06-20 09:14:31 -0700 · Steve

Files touched

• M ingest-browserbase.js

Diff

commit 18fb32f20fa6aa19f46ec0aa349a872329aec32f
Author: Steve <steve@designerwallcoverings.com>
Date:   Sat Jun 20 09:14:31 2026 -0700

    Vendor-IG: multi-mirror scraper (picuki/picnob/pixwox); keep only cacheable-image posts (picnob URLs ephemeral)
---
 ingest-browserbase.js | 51 +++++++++++++++++++++++++++++++++++----------------
 1 file changed, 35 insertions(+), 16 deletions(-)

diff --git a/ingest-browserbase.js b/ingest-browserbase.js
index b83e9a0..012c38e 100644
--- a/ingest-browserbase.js
+++ b/ingest-browserbase.js
@@ -31,28 +31,30 @@ function pickHandles(argv) {
   return all.filter(h => want.has(h.handle.toLowerCase()));
 }
 
-// IG itself login-walls public profiles; Picuki renders them without login and
-// is scrapable via the cloud browser. We pull each post's thumbnail (Picuki CDN
-// proxies the real IG image), its caption, and the Picuki media permalink.
-async function scrapeProfile(page, handle) {
-  const resp = await page.goto(`https://www.picuki.com/profile/${handle}`, { waitUntil: 'domcontentloaded', timeout: 45000 });
-  if (resp && resp.status() >= 400) throw new Error('picuki ' + resp.status());
-  await page.waitForTimeout(3000);
-  try { await page.click('text=/Accept|Agree|Consent/i', { timeout: 2000 }); } catch {}
-  // scroll to trigger lazy-loaded thumbnails, then settle
+// IG login-walls public profiles; Picuki-family mirrors render them without
+// login and are scrapable via the cloud browser (they proxy the real IG image).
+// They're clones with the same DOM, so one extractor works across all of them.
+// We try each mirror until one yields posts.
+const MIRRORS = [
+  h => `https://www.picuki.com/profile/${h}`,
+  h => `https://www.picnob.com/profile/${h}/`,
+  h => `https://www.pixwox.com/profile/${h}/`
+];
+async function extractPosts(page, max) {
+  try { await page.click('text=/Accept|Agree|Consent/i', { timeout: 1500 }); } catch {}
   try { await page.evaluate(() => window.scrollBy(0, 1600)); } catch {}
-  await page.waitForTimeout(2500);
+  await page.waitForTimeout(2200);
   return await page.evaluate((max) => {
-    const out = [];
-    const seen = new Set();
-    document.querySelectorAll('a[href*="/media/"]').forEach(a => {
+    const out = []; const seen = new Set();
+    // picuki uses /media/<id>; picnob/pixwox use /post/<id> or /p/<code>
+    document.querySelectorAll('a[href*="/media/"], a[href*="/post/"], a[href*="/p/"]').forEach(a => {
       const href = a.href;
-      const id = (href.match(/\/media\/(\d+)/) || [])[1];
+      const id = (href.match(/\/(?:media|post|p)\/([A-Za-z0-9_-]+)/) || [])[1];
       if (!id || seen.has(id)) return;
       const img = a.querySelector('img') || a.parentElement.querySelector('img');
       const src = img && (img.getAttribute('src') || (img.getAttribute('srcset') || '').split(' ')[0]);
       if (!src || !/^https?:/.test(src)) return;
-      const box = a.closest('.box-photo, .item, li, div');
+      const box = a.closest('.box-photo, .item, li, article, div');
       const desc = box && box.querySelector('.photo-description, .description, .caption');
       seen.add(id);
       out.push({
@@ -62,7 +64,24 @@ async function scrapeProfile(page, handle) {
       });
     });
     return out.slice(0, max);
-  }, MAX_POSTS);
+  }, max);
+}
+async function scrapeProfile(page, handle) {
+  let lastErr = 'no mirror yielded posts';
+  for (const m of MIRRORS) {
+    try {
+      const resp = await page.goto(m(handle), { waitUntil: 'domcontentloaded', timeout: 40000 });
+      if (resp && resp.status() >= 400) { lastErr = 'mirror ' + resp.status(); continue; }
+      await page.waitForTimeout(2800);
+      if (/no posts|user has no|not found/i.test(await page.evaluate(() => document.body.innerText.slice(0, 400)))) {
+        lastErr = 'empty profile'; continue;
+      }
+      const posts = await extractPosts(page, MAX_POSTS);
+      if (posts.length) return posts;
+      lastErr = '0 posts extracted';
+    } catch (e) { lastErr = e.message.slice(0, 50); }
+  }
+  throw new Error(lastErr);
 }
 
 (async () => {

← 5d0e46a Compose deep-link: accept ?compose/title/img/link/vendor to  ·  back to Dw Launches  ·  Vendor-IG: in-session screenshot capture (DTD verdict B) — + 51a9575 →