`onboarding/sangetsu-lilycolor/scripts/supervise.sh`

37 lines
#!/usr/bin/env bash
# supervise.sh — keep a resumable local-LLM job alive across external kills.
#
# Mac2 memory pressure (swap thrash) periodically jetsam-kills the enrichment /
# translation python processes mid-run. The wrapped scripts are RESUMABLE and
# HARDENED (retry/defer on transient ollama-down or Henry-unmount, never burn a
# SKU), so a kill costs seconds, not data. This wrapper relaunches within ~5s of
# any non-clean exit until the job genuinely completes.
#
# Completion signal = a CLEAN exit (rc==0) that produced NO new output rows.
#   - A kill (jetsam/SIGKILL) yields rc!=0  -> always relaunch.
#   - A productive pass yields rc==0 + new rows -> loop to continue remaining work.
#   - A pass that exits clean with nothing new -> the job is done -> stop.
# This needs no fragile log-marker matching and works for both jobs identically.
#
# Usage: supervise.sh <label> <script-rel-path> <output-file> <log-file>
set -u
LABEL="$1"; SCRIPT="$2"; OUT="$3"; LOG="$4"
HERE="$(cd "$(dirname "$0")/.." && pwd)"
cd "$HERE" || exit 1

count() { wc -l < "$OUT" 2>/dev/null | tr -d ' ' || echo 0; }

echo "[$(date +%H:%M:%S)] supervisor($LABEL) start" >> "$LOG"
while true; do
  before="$(count)"
  python3 "$SCRIPT" >> "$LOG" 2>&1
  rc=$?
  after="$(count)"
  if [ "$rc" -eq 0 ] && [ "$before" = "$after" ]; then
    echo "[$(date +%H:%M:%S)] supervisor($LABEL) COMPLETE (clean exit, no new rows at ${after}) — stopping" >> "$LOG"
    break
  fi
  echo "[$(date +%H:%M:%S)] supervisor($LABEL) pass ended rc=$rc rows ${before}->${after} — relaunch in 5s" >> "$LOG"
  sleep 5
done