[object Object]

← back to Norma

instagram-agent: harden getJSON (response-stream error + retry, fixes mid-body ECONNRESET crash) + backfill-all.js driver; gitignore regenerable live-media caches

ea7bf527773915541ee0c5b67fba6f98b575d676 · 2026-08-18 12:26:34 -0700 · Steve Abrams

Files touched

Diff

commit ea7bf527773915541ee0c5b67fba6f98b575d676
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Tue Aug 18 12:26:34 2026 -0700

    instagram-agent: harden getJSON (response-stream error + retry, fixes mid-body ECONNRESET crash) + backfill-all.js driver; gitignore regenerable live-media caches
---
 agents/instagram-agent/.gitignore      |  1 +
 agents/instagram-agent/backfill-all.js | 43 ++++++++++++++++++++++++++++++++++
 agents/instagram-agent/live-media.js   | 25 ++++++++++++++++----
 3 files changed, 65 insertions(+), 4 deletions(-)

diff --git a/agents/instagram-agent/.gitignore b/agents/instagram-agent/.gitignore
index d66debc..8e7a1a6 100644
--- a/agents/instagram-agent/.gitignore
+++ b/agents/instagram-agent/.gitignore
@@ -1,3 +1,4 @@
 .env
 .env.*
 data/credentials/
+data/live-media*.json
diff --git a/agents/instagram-agent/backfill-all.js b/agents/instagram-agent/backfill-all.js
new file mode 100644
index 0000000..ceb808e
--- /dev/null
+++ b/agents/instagram-agent/backfill-all.js
@@ -0,0 +1,43 @@
+/**
+ * backfill-all.js — one-off: paginate the FULL post history of every IG account into its
+ * per-account cache (data/live-media[-<id>].json). Read-only Graph /media sweep, $0.
+ * Sequential with a delay between accounts to stay under Meta's rate limit. Accounts that
+ * are already `complete` are skipped (DW is already fully backfilled). Continues on error
+ * and prints a per-account summary; re-run to retry any failures.
+ */
+require('dotenv').config();
+const fs = require('fs');
+const path = require('path');
+const { listAccounts, refreshLiveMedia } = require('./live-media');
+
+const DATA = path.join(__dirname, 'data');
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+const cachePath = (id) => id === process.env.IG_USER_ID
+  ? path.join(DATA, 'live-media.json') : path.join(DATA, `live-media-${id}.json`);
+const isComplete = (id) => { try { return !!JSON.parse(fs.readFileSync(cachePath(id), 'utf8')).complete; } catch { return false; } };
+
+(async () => {
+  const accts = listAccounts();
+  const todo = accts.filter((a) => !isComplete(a.ig_user_id));
+  console.log(`[backfill-all] ${accts.length} accounts · ${accts.length - todo.length} already complete · ${todo.length} to backfill`);
+  const results = [];
+  let i = 0;
+  for (const a of todo) {
+    i++;
+    process.stdout.write(`[${i}/${todo.length}] @${a.handle} … `);
+    try {
+      const r = await refreshLiveMedia({ igUserId: a.ig_user_id, full: true });
+      console.log(`+${r.added} (${r.scanned} scanned, ${r.pages}p)${r.complete ? ' ✓complete' : ' (capped)'}`);
+      results.push({ handle: a.handle, added: r.added, scanned: r.scanned, complete: r.complete, ok: true });
+    } catch (e) {
+      console.log(`ERROR: ${e.message}`);
+      results.push({ handle: a.handle, error: e.message, ok: false });
+    }
+    await sleep(1500); // be gentle on the Graph rate limit
+  }
+  const ok = results.filter((r) => r.ok);
+  const tot = ok.reduce((n, r) => n + (r.added || 0), 0);
+  console.log(`\n[backfill-all] DONE: ${ok.length}/${results.length} ok · +${tot.toLocaleString()} posts cached`);
+  const fails = results.filter((r) => !r.ok);
+  if (fails.length) console.log('[backfill-all] FAILURES (re-run to retry): ' + fails.map((f) => `@${f.handle}: ${f.error}`).join(' · '));
+})();
diff --git a/agents/instagram-agent/live-media.js b/agents/instagram-agent/live-media.js
index 6326bd7..6a92100 100644
--- a/agents/instagram-agent/live-media.js
+++ b/agents/instagram-agent/live-media.js
@@ -61,16 +61,33 @@ function resolveAccount(igUserId) {
   return roster.find((r) => r.ig_user_id === id) || null; // null = not whitelisted → reject
 }
 
-// ── tiny GET-JSON (no deps) ───────────────────────────────────────────────────
-function getJSON(url) {
+// ── tiny GET-JSON (no deps) — handles BOTH request- and response-stream errors
+// (a mid-body ECONNRESET emits 'error' on the response; without a listener it crashes
+// the process), with a small retry so transient socket resets self-heal. ──────────────
+function getJSONOnce(url) {
   return new Promise((resolve, reject) => {
-    https.get(url, (r) => {
+    const req = https.get(url, { timeout: 30000 }, (r) => {
       let d = '';
       r.on('data', (c) => (d += c));
       r.on('end', () => { try { resolve(JSON.parse(d)); } catch (e) { reject(e); } });
-    }).on('error', reject);
+      r.on('error', reject); // ← response-stream error (mid-body reset) — was unhandled
+    });
+    req.on('error', reject);
+    req.on('timeout', () => req.destroy(new Error('request timeout')));
   });
 }
+async function getJSON(url, tries = 3) {
+  let last;
+  for (let i = 0; i < tries; i++) {
+    try { return await getJSONOnce(url); }
+    catch (e) {
+      last = e;
+      if (!/ECONNRESET|timeout|ETIMEDOUT|EAI_AGAIN|socket hang up/i.test(e.message)) break; // only retry transient net errors
+      await new Promise((r) => setTimeout(r, 800 * (i + 1)));
+    }
+  }
+  throw last;
+}
 
 // ── tombstones (shared identity with posts-api: permalink is the stable id) ────
 function readTombstones() {

← 8510437 auto-data-snapshot: 2026-08-18T12:24:13 (1 data files) — age  ·  back to Norma  ·  auto-data-snapshot: 2026-08-18T12:56:03 (1 data files) — age d86e027 →