← back to Wallco Ai
refiner_compare: two-machine split — CMP_SHARD/CMP_SHARDS env shards indices across ComfyUI instances; progress.json now disk-derived so both shards write it safely; per-shard watchdog + .done sentinel
9b5ec50fb10cf7c4ecfe722594efaa2fc64ef17f · 2026-05-19 12:30:29 -0700 · SteveStudio2
Files touched
M scripts/refiner_compare.jsM scripts/refiner_compare_watchdog.sh
Diff
commit 9b5ec50fb10cf7c4ecfe722594efaa2fc64ef17f
Author: SteveStudio2 <steve@designerwallcoverings.com>
Date: Tue May 19 12:30:29 2026 -0700
refiner_compare: two-machine split — CMP_SHARD/CMP_SHARDS env shards indices across ComfyUI instances; progress.json now disk-derived so both shards write it safely; per-shard watchdog + .done sentinel
---
scripts/refiner_compare.js | 83 ++++++++++++++++++++++++-------------
scripts/refiner_compare_watchdog.sh | 27 ++++++++----
2 files changed, 73 insertions(+), 37 deletions(-)
diff --git a/scripts/refiner_compare.js b/scripts/refiner_compare.js
index 92275cc..98bc413 100644
--- a/scripts/refiner_compare.js
+++ b/scripts/refiner_compare.js
@@ -91,54 +91,81 @@ function render(workflow, outPath) {
return false;
}
+// --- sharding: split the 25 pairs across N machines, one ComfyUI per shard.
+// shard k of n renders the indices where i % n === k. COMFY_URL picks which
+// ComfyUI (and therefore which GPU) this shard talks to. Default = single box.
+const SHARDS = Math.max(1, parseInt(process.env.CMP_SHARDS || '1', 10));
+const SHARD = Math.min(Math.max(0, parseInt(process.env.CMP_SHARD || '0', 10)), SHARDS - 1);
+const mine = i => SHARDS === 1 || i % SHARDS === SHARD;
+const TAG = SHARDS > 1 ? `[shard ${SHARD}/${SHARDS}] ` : '';
+
(async () => {
const t0 = Date.now();
- // Every row built up-front so the page shows all 25 slots immediately.
- const rows = STYLES.map((style, i) => ({
- i, style, brand: FASHION_PALETTES[i % FASHION_PALETTES.length].brand,
- seed: 770000 + i, base: null, refiner: null,
- }));
- const TOTAL = STYLES.length * 2; // 50 renders: 25 base-only + 25 refiner
- let nDone = 0;
- const save = () => fs.writeFileSync(path.join(OUT, 'progress.json'),
- JSON.stringify({ done: nDone, total: TOTAL, rows }, null, 1));
- // resume-safe: a render that already produced a real file is skipped, so a
- // crash + restart picks up where it left off instead of re-rendering pair 0.
+ const TOTAL = STYLES.length * 2; // 50 renders total across all shards
const haveImg = p => { try { return fs.statSync(p).size > 1000; } catch { return false; } };
+ // progress.json is derived fresh from disk on every write, so multiple shard
+ // processes can each write it without clobbering — the last writer is always
+ // correct. Resume-safety also keys off these same on-disk files.
+ const save = () => {
+ const rows = STYLES.map((style, i) => ({
+ i, style, brand: FASHION_PALETTES[i % FASHION_PALETTES.length].brand, seed: 770000 + i,
+ base: haveImg(path.join(OUT, `${i}_base.png`)) ? `${i}_base.png` : null,
+ refiner: haveImg(path.join(OUT, `${i}_refiner.png`)) ? `${i}_refiner.png` : null,
+ }));
+ const done = rows.reduce((n, r) => n + (r.base ? 1 : 0) + (r.refiner ? 1 : 0), 0);
+ fs.writeFileSync(path.join(OUT, 'progress.json'), JSON.stringify({ done, total: TOTAL, rows }, null, 1));
+ };
+ console.log(`${TAG}COMFY=${COMFY}`);
save();
- // PASS 1 — every base-only render. Grouping all 25 together keeps ComfyUI's
- // base checkpoint hot the whole pass instead of swapping it out each pair.
+ // PASS 1 — every base-only render this shard owns. Grouping bases keeps
+ // ComfyUI's base checkpoint hot instead of swapping it out each pair.
for (let i = 0; i < STYLES.length; i++) {
+ if (!mine(i)) continue;
const style = STYLES[i], pal = FASHION_PALETTES[i % FASHION_PALETTES.length];
const seed = 770000 + i, POS = buildPos(style, pal);
- const baseFile = `${i}_base.png`, basePath = path.join(OUT, baseFile);
- if (haveImg(basePath)) { rows[i].base = baseFile; console.log(` ${i} base — skip (already rendered)`); }
+ const basePath = path.join(OUT, `${i}_base.png`);
+ if (haveImg(basePath)) { console.log(` ${i} base — skip (already rendered)`); }
else {
- try { rows[i].base = render(baseWorkflow(seed, POS), basePath) ? baseFile : false; }
- catch (e) { rows[i].base = false; console.log(` ${i} base ERR ${e.message}`); }
+ try { render(baseWorkflow(seed, POS), basePath); }
+ catch (e) { console.log(` ${i} base ERR ${e.message}`); }
}
- nDone++; save();
- console.log(`[base ${i + 1}/25] ${style} · ${pal.brand} — ${rows[i].base ? 'ok' : 'FAIL'} · ${((Date.now() - t0) / 60000).toFixed(1)}min`);
+ save();
+ console.log(`${TAG}[base ${i + 1}/25] ${style} · ${pal.brand} — ${haveImg(basePath) ? 'ok' : 'FAIL'} · ${((Date.now() - t0) / 60000).toFixed(1)}min`);
}
- // PASS 2 — every base+refiner render, base+refiner checkpoints stay hot.
+ // PASS 2 — every base+refiner render this shard owns.
for (let i = 0; i < STYLES.length; i++) {
+ if (!mine(i)) continue;
const style = STYLES[i], pal = FASHION_PALETTES[i % FASHION_PALETTES.length];
const seed = 770000 + i, POS = buildPos(style, pal);
- const refFile = `${i}_refiner.png`, refPath = path.join(OUT, refFile);
- if (haveImg(refPath)) { rows[i].refiner = refFile; console.log(` ${i} refiner — skip (already rendered)`); }
+ const refPath = path.join(OUT, `${i}_refiner.png`);
+ if (haveImg(refPath)) { console.log(` ${i} refiner — skip (already rendered)`); }
else {
- try { rows[i].refiner = render(refinerWorkflow(seed, POS), refPath) ? refFile : false; }
- catch (e) { rows[i].refiner = false; console.log(` ${i} refiner ERR ${e.message}`); }
+ try { render(refinerWorkflow(seed, POS), refPath); }
+ catch (e) { console.log(` ${i} refiner ERR ${e.message}`); }
}
- nDone++; save();
- console.log(`[refiner ${i + 1}/25] ${style} · ${pal.brand} — ${rows[i].refiner ? 'ok' : 'FAIL'} · ${((Date.now() - t0) / 60000).toFixed(1)}min`);
+ save();
+ console.log(`${TAG}[refiner ${i + 1}/25] ${style} · ${pal.brand} — ${haveImg(refPath) ? 'ok' : 'FAIL'} · ${((Date.now() - t0) / 60000).toFixed(1)}min`);
}
- // compare page
+ // sentinel — this shard finished its slice; the watchdog reads it to know
+ // not to restart this shard while the OTHER shard is still rendering.
+ fs.writeFileSync(path.join(OUT, `.shard${SHARD}.done`), new Date().toISOString());
+
+ // static compare page — written only once EVERY render is on disk, so the
+ // first shard to finish doesn't replace the live page out from under the other.
+ const finalRows = STYLES.map((style, i) => ({
+ i, style, brand: FASHION_PALETTES[i % FASHION_PALETTES.length].brand, seed: 770000 + i,
+ base: haveImg(path.join(OUT, `${i}_base.png`)) ? `${i}_base.png` : false,
+ refiner: haveImg(path.join(OUT, `${i}_refiner.png`)) ? `${i}_refiner.png` : false,
+ }));
+ if (!finalRows.every(r => r.base && r.refiner)) {
+ console.log(`${TAG}shard done; other shard(s) still rendering — leaving live page up.`);
+ return;
+ }
const esc = s => String(s).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
- const cards = rows.map(r => `
+ const cards = finalRows.map(r => `
<div class="row">
<div class="meta"><b>#${r.i + 1}</b> · ${esc(r.style)} · <span class="brand">${esc(r.brand)}</span> · seed ${r.seed}</div>
<div class="pair">
diff --git a/scripts/refiner_compare_watchdog.sh b/scripts/refiner_compare_watchdog.sh
index b225dae..de28150 100755
--- a/scripts/refiner_compare_watchdog.sh
+++ b/scripts/refiner_compare_watchdog.sh
@@ -1,25 +1,34 @@
#!/bin/bash
-# Watchdog for refiner_compare.js — restarts it if it dies, exits when the
-# 25-pair job completes. The script itself is resume-safe (skips rendered
-# images), so a restart picks up where the crash left off.
+# Watchdog for ONE refiner_compare shard. Reads CMP_SHARD / CMP_SHARDS / COMFY_URL
+# from the environment, restarts its shard if it dies, and exits when the shard
+# writes its .done sentinel (or all 50 renders are on disk). The shard process is
+# launched with a "shardN" arg purely so pgrep can tell the two shards apart.
cd "$HOME/Projects/wallco-ai" || exit 1
+SHARD="${CMP_SHARD:-0}"
+TAG="shard${SHARD}"
PROG="public/refiner-compare/progress.json"
-LOG=/tmp/refiner_compare.log
+SENTINEL="public/refiner-compare/.${TAG}.done"
+LOG="/tmp/refiner_compare.${TAG}.log"
restarts=0
+echo "$(date '+%H:%M:%S') $TAG watchdog up — CMP_SHARDS=${CMP_SHARDS:-1} COMFY_URL=${COMFY_URL:-default}"
while true; do
+ if [ -f "$SENTINEL" ] && ! pgrep -f "refiner_compare.js $TAG" >/dev/null 2>&1; then
+ echo "$(date '+%H:%M:%S') $TAG finished (sentinel present) — watchdog exiting after $restarts restart(s)"
+ break
+ fi
done=$(python3 -c "import json;print(json.load(open('$PROG'))['done'])" 2>/dev/null || echo 0)
if [ "${done:-0}" -ge 50 ]; then
- echo "$(date '+%H:%M:%S') job complete ($done/50 renders) — watchdog exiting after $restarts restart(s)"
+ echo "$(date '+%H:%M:%S') all 50 renders done — $TAG watchdog exiting"
break
fi
- if ! pgrep -f 'scripts/refiner_compare.js' >/dev/null 2>&1; then
+ if ! pgrep -f "refiner_compare.js $TAG" >/dev/null 2>&1; then
if [ "$restarts" -ge 30 ]; then
- echo "$(date '+%H:%M:%S') hit 30-restart cap at $done/50 — giving up, check $LOG"
+ echo "$(date '+%H:%M:%S') $TAG hit 30-restart cap at $done/50 — giving up, check $LOG"
break
fi
restarts=$((restarts+1))
- echo "$(date '+%H:%M:%S') refiner_compare.js not running (at $done/50 renders) — restart #$restarts"
- nohup node scripts/refiner_compare.js >> "$LOG" 2>&1 &
+ echo "$(date '+%H:%M:%S') $TAG not running (at $done/50 renders) — restart #$restarts"
+ nohup node scripts/refiner_compare.js "$TAG" >> "$LOG" 2>&1 &
sleep 5
fi
sleep 30
← f042d90 refiner_compare: reorder into base pass then refiner pass —
·
back to Wallco Ai
·
deploy-kamatera: image sync is incremental + non-destructive 9829d20 →