← back to Designer Wallcoverings
import drain self-heals: backend-down never burns item attempts (refund+heal via ensure-import-app.sh), driver resurrects infra-failed items each pass and never exits on drained, :9830 app now pm2-supervised; 1027 brewster items requeued
84bb022a853209f22c33da575dfd1f02a8a4bd2f · 2026-07-13 08:27:17 -0700 · Steve
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Files touched
M DW-Programming/ImportNewSkufromURL/scripts/drain-all-queues.shM DW-Programming/ImportNewSkufromURL/scripts/import-queue-runner.jsA DW-Programming/ImportNewSkufromURL/scripts/resurrect-infra-failed.js
Diff
commit 84bb022a853209f22c33da575dfd1f02a8a4bd2f
Author: Steve <steve@designerwallcoverings.com>
Date: Mon Jul 13 08:27:17 2026 -0700
import drain self-heals: backend-down never burns item attempts (refund+heal via ensure-import-app.sh), driver resurrects infra-failed items each pass and never exits on drained, :9830 app now pm2-supervised; 1027 brewster items requeued
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
.../scripts/drain-all-queues.sh | 30 ++++++++++--
.../scripts/import-queue-runner.js | 53 ++++++++++++++++++++--
.../scripts/resurrect-infra-failed.js | 39 ++++++++++++++++
3 files changed, 115 insertions(+), 7 deletions(-)
diff --git a/DW-Programming/ImportNewSkufromURL/scripts/drain-all-queues.sh b/DW-Programming/ImportNewSkufromURL/scripts/drain-all-queues.sh
index 57125a4e..2079b70b 100755
--- a/DW-Programming/ImportNewSkufromURL/scripts/drain-all-queues.sh
+++ b/DW-Programming/ImportNewSkufromURL/scripts/drain-all-queues.sh
@@ -40,6 +40,15 @@ echo "[drain-all] starting $(date) pid=$$" >> "$LOG"
while true; do
if [ -f "$STOP" ]; then echo "[drain-all] STOP file present, exiting $(date)" >> "$LOG"; exit 0; fi
THROTTLED=0
+ UNAVAILABLE=0
+ # SELF-HEAL 1: requeue anything that "failed" for infra reasons (backend down /
+ # connection drops) — an outage must never permanently strand items as failed.
+ node scripts/resurrect-infra-failed.js "$DIR" >> "$LOG" 2>&1
+ # SELF-HEAL 2: make sure the :9830 app answers before spending a pass on it.
+ if ! sh scripts/ensure-import-app.sh >> "$LOG" 2>&1; then
+ echo "[drain-all] :9830 unhealable — sleeping 10m before retry $(date)" >> "$LOG"
+ sleep 600; continue
+ fi
# ENRICHED-FIRST order (Steve 2026-06-10): the 5 vendors with full-page
# all-images + spec sidecars drain ahead of brewster/york, smallest→largest so
# complete catalogs land fastest. Any other *-import-queue.jsonl follows.
@@ -61,14 +70,27 @@ while true; do
echo "[drain-all] throttle detected (rc=75) — abandoning pass early $(date)" >> "$LOG"
break
fi
+ # rc=69 (EX_UNAVAILABLE): runner couldn't heal the backend — stop the pass,
+ # sleep short, re-enter the loop (which re-heals) instead of grinding on.
+ if [ "$RC" -eq 69 ]; then
+ UNAVAILABLE=1
+ echo "[drain-all] backend unavailable (rc=69) — abandoning pass $(date)" >> "$LOG"
+ break
+ fi
done
PENDING=$(count_pending)
- if [ "$PENDING" = "0" ]; then
- echo "[drain-all] ALL QUEUES DRAINED $(date)" >> "$LOG"; exit 0
- fi
- if [ "$THROTTLED" -eq 1 ]; then
+ # NEVER-STOP RULE (Steve 2026-07-13): 0 pending is a resting state, not an exit.
+ # Queues gain items from new reconciles, and the resurrect step can repopulate
+ # after an outage. The ONLY intentional exit is the .drain-STOP file.
+ if [ "$UNAVAILABLE" -eq 1 ]; then
+ echo "[drain-all] backend down — $PENDING pending, sleeping 10m $(date)" >> "$LOG"
+ sleep 600
+ elif [ "$THROTTLED" -eq 1 ]; then
echo "[drain-all] variant throttle hit — $PENDING pending, sleeping 2h $(date)" >> "$LOG"
sleep 7200
+ elif [ "$PENDING" = "0" ]; then
+ echo "[drain-all] all queues drained — idling 30m, watching for new work $(date)" >> "$LOG"
+ sleep 1800
else
echo "[drain-all] pass complete — $PENDING pending, sleeping 30m $(date)" >> "$LOG"
sleep 1800
diff --git a/DW-Programming/ImportNewSkufromURL/scripts/import-queue-runner.js b/DW-Programming/ImportNewSkufromURL/scripts/import-queue-runner.js
index 7ce6b091..9dc799c7 100644
--- a/DW-Programming/ImportNewSkufromURL/scripts/import-queue-runner.js
+++ b/DW-Programming/ImportNewSkufromURL/scripts/import-queue-runner.js
@@ -95,6 +95,21 @@ function post(path, body, ms) {
});
}
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+// ---- BACKEND SELF-HEAL (born 2026-07-13) ----
+// On 2026-07-04 the :9830 app died and 1,027 brewster items burned all 6 retry
+// attempts against ECONNREFUSED — an infra failure charged to the items. Rule:
+// backend-down NEVER consumes an item's attempt budget. We heal the app via the
+// shared ensure-import-app.sh (pm2 restart/start + poll); if it can't come back,
+// exit 69 (EX_UNAVAILABLE) so the drain driver sleeps and retries the whole pass.
+const { execFileSync } = require('child_process');
+const BACKEND_DOWN_RE = /ECONNREFUSED|ECONNRESET|socket hang up/i;
+function healBackend() {
+ try {
+ execFileSync('/bin/sh', [require('path').join(__dirname, 'ensure-import-app.sh')], { stdio: 'inherit', timeout: 180000 });
+ return true;
+ } catch { return false; }
+}
function load() { return fs.readFileSync(QUEUE, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l)); }
function save(items) { fs.writeFileSync(QUEUE, items.map(x => JSON.stringify(x)).join('\n') + '\n'); }
@@ -182,6 +197,12 @@ if (!QUEUE || !fs.existsSync(QUEUE)) { console.error('usage: import-queue-runner
let items = load();
const pending = items.filter(x => x.status === 'pending');
console.log(`[import-queue] ${QUEUE}: ${pending.length} pending / ${items.length} total — draining up to ${MAX}`);
+ // Gate the whole pass on a live backend (ensure-import-app.sh is a no-op when
+ // :9830 already answers). Never touch a single item against a dead app.
+ if (pending.length && !healBackend()) {
+ console.log('[import-queue] backend down at pass start and unhealable — exiting 69 (EX_UNAVAILABLE)');
+ process.exit(69);
+ }
let created = 0, failed = 0, dupSkip = 0, processed = 0, throttled = false, consecFail = 0;
for (const it of items) {
@@ -207,9 +228,23 @@ if (!QUEUE || !fs.existsSync(QUEUE)) { console.error('usage: import-queue-runner
preview = scPrev;
console.log(` ⓢ ${it.sku}: import/sku ${note} → built from sidecar (${scPrev.images.length} imgs)`);
} else {
- // TRANSIENT network drops (next-dev recompiles mid-request → "socket
- // hang up") must NOT permanently fail on one shot — leave pending.
- const transient = /socket hang up|ECONNRESET|ECONNREFUSED|timeout/i.test(note);
+ // BACKEND DOWN (connection-level): not the item's fault — refund the
+ // attempt, leave pending, heal the app. Unhealable → exit 69 so the
+ // driver retries the pass later instead of grinding the queue to failed.
+ if (BACKEND_DOWN_RE.test(note)) {
+ it.attempts--;
+ it.note = note + ' (backend down — attempt refunded)';
+ console.log(` ⚕ ${it.sku}: ${note} — backend down, healing (attempt not charged)`);
+ save(items);
+ if (!healBackend()) {
+ console.log('[import-queue] backend unhealable — exiting 69 (EX_UNAVAILABLE)');
+ process.exit(69);
+ }
+ continue; // same pass resumes with the next pending item
+ }
+ // TRANSIENT one-off drops (e.g. request timeout mid-recompile) must NOT
+ // permanently fail on one shot — leave pending.
+ const transient = /timeout/i.test(note);
if (transient && (it.attempts || 0) < 6) {
it.note = note + ' (left pending for retry)';
console.log(` ↻ ${it.sku}: ${note} — left pending (attempt ${it.attempts})`);
@@ -231,6 +266,18 @@ if (!QUEUE || !fs.existsSync(QUEUE)) { console.error('usage: import-queue-runner
// completes server-side → a wasted dedup-skip retry next pass. Give it room.
const cr = await post('/api/create-product', preview, 180000).catch(e => ({ err: e.message }));
const errStr = cr.err || cr.json?.error || '';
+ // Backend died between extract and create — same refund-and-heal rule.
+ if (cr.err && BACKEND_DOWN_RE.test(errStr)) {
+ it.attempts--;
+ it.note = errStr + ' (backend down — attempt refunded)';
+ console.log(` ⚕ ${it.sku}: ${errStr} — backend down at create, healing (attempt not charged)`);
+ save(items);
+ if (!healBackend()) {
+ console.log('[import-queue] backend unhealable — exiting 69 (EX_UNAVAILABLE)');
+ process.exit(69);
+ }
+ continue;
+ }
if (/VARIANT_THROTTLE_EXCEEDED|variant creation limit/i.test(errStr)) {
it.attempts--; // not the item's fault — leave fully pending
throttled = true; console.log(` 🛑 ${it.sku}: Shopify daily variant cap reached — stopping, ${items.filter(x=>x.status==='pending').length} still queued`);
diff --git a/DW-Programming/ImportNewSkufromURL/scripts/resurrect-infra-failed.js b/DW-Programming/ImportNewSkufromURL/scripts/resurrect-infra-failed.js
new file mode 100644
index 00000000..551a7bdb
--- /dev/null
+++ b/DW-Programming/ImportNewSkufromURL/scripts/resurrect-infra-failed.js
@@ -0,0 +1,39 @@
+#!/usr/bin/env node
+/**
+ * resurrect-infra-failed.js — requeue queue items that "failed" for INFRA reasons
+ * (backend down / connection drops), not item reasons. Runs at the start of every
+ * drain pass so a backend outage can never permanently strand items as failed.
+ *
+ * Infra notes look like: "connect ECONNREFUSED 127.0.0.1:9830", "ECONNRESET",
+ * "socket hang up", "timeout". Genuine item failures ("source 404", "Title became
+ * empty…") are left alone. Resurrected items get status=pending, attempts=0.
+ *
+ * Usage: node scripts/resurrect-infra-failed.js [dir=data/scraper-audit/reconcile]
+ */
+const fs = require('fs');
+const path = require('path');
+
+const DIR = process.argv[2] || 'data/scraper-audit/reconcile';
+const INFRA_RE = /ECONNREFUSED|ECONNRESET|socket hang up|backend down|timeout/i;
+
+let total = 0;
+for (const f of fs.readdirSync(DIR).filter(f => f.endsWith('-import-queue.jsonl'))) {
+ const p = path.join(DIR, f);
+ const items = fs.readFileSync(p, 'utf8').split('\n').filter(Boolean).map(l => JSON.parse(l));
+ let n = 0;
+ for (const it of items) {
+ if (it.status === 'failed' && INFRA_RE.test(it.note || '')) {
+ it.status = 'pending';
+ it.attempts = 0;
+ delete it.doneAt;
+ it.note = `resurrected ${new Date().toISOString().slice(0, 10)} (was infra-failed)`;
+ n++;
+ }
+ }
+ if (n) {
+ fs.writeFileSync(p, items.map(x => JSON.stringify(x)).join('\n') + '\n');
+ console.log(`[resurrect] ${f}: ${n} infra-failed → pending`);
+ total += n;
+ }
+}
+console.log(`[resurrect] total resurrected: ${total}`);
← d90232b2 auto-save: 2026-07-13T08:23:39 (4 files) — pending-approval/
·
back to Designer Wallcoverings
·
drain self-heal round 2: missing playwright browsers treated 615561a2 →