`onboarding/sangetsu-lilycolor/scripts/mac2-guarded.sh`

77 lines
#!/bin/bash
# Memory-SAFE Lilycolor enrichment on Mac2 ollama (the only box whose llama-server
# actually handles this vision model — Mac1 hangs on real images). A swap-guard paces
# requests: the enrich driver is SIGSTOP'd whenever Mac2 swap headroom is low and
# SIGCONT'd only when there's enough room to absorb one inference spike (~1.1G observed).
# => never forces inference under pressure => never triggers the low-swap watchdog reboot.
# Makes progress opportunistically (e.g. as parallel claude sessions close / load drops).
# Fully local ($0), resumable, reversible (JSONL only). Reads images from local disk copy.
set -u
PROJ="$HOME/Projects/Designer-Wallcoverings/onboarding/sangetsu-lilycolor"; cd "$PROJ" || exit 1
ST="$PROJ/staging"
LOCAL_IMG="$HOME/dw-lily-images-local"
GOOD="$ST/enrichment-full-063026A.good-seed.jsonl"
REC="$ST/enrichment-full-063026A.guarded-recover.jsonl"
FINAL="$ST/enrichment-full-063026A.final.jsonl"
CONT_ABOVE_MB=2200   # only allow a new inference when swap free > this
STOP_BELOW_MB=1000    # pause the driver when swap free < this
log(){ echo "[guard $(date '+%H:%M:%S')] $*" >> "$ST/enrich-guard.log"; }
swap_free_mb(){ sysctl -n vm.swapusage 2>/dev/null | sed -E 's/.*free = ([0-9.]+)M.*/\1/' | cut -d. -f1; }

[ -d "$LOCAL_IMG" ] || { log "FATAL local img dir missing"; exit 1; }
[ -f "$GOOD" ] || cp "$FINAL" "$GOOD"
log "start. good-seed=$(wc -l < "$GOOD"|tr -d ' ') local-imgs=$(ls -1 "$LOCAL_IMG"|wc -l|tr -d ' ')"

# WAIT for headroom BEFORE launching the driver (no fragile mid-startup SIGSTOP).
# When Mac2 is loaded (parallel claude sessions + ollama), just wait — launch only
# once there's room to absorb an inference spike. This is the safe idle state.
while true; do
  fm=$(swap_free_mb); fm=${fm:-0}
  [ "$fm" -gt "$CONT_ABOVE_MB" ] && break
  log "waiting for headroom (swap free ${fm}M < ${CONT_ABOVE_MB}M)"
  sleep 30
done
log "headroom OK (swap free $(swap_free_mb)M) — launching driver"
ENRICH_HENRY_DIR="$LOCAL_IMG" ENRICH_SANGETSU_DIR="/nonexistent" ENRICH_SHARD_COUNT=1 \
ENRICH_OLLAMA_URL=http://127.0.0.1:11434 ENRICH_VL_MAX=768 ENRICH_DONE_SEEDS="$GOOD" \
python3 -u scripts/enrich-full.py "$REC" >> "$ST/enrich-guard.log" 2>&1 &
DPID=$!
log "driver PID $DPID launched (paced by swap guard)"

paused=0
while kill -0 "$DPID" 2>/dev/null; do
  fm=$(swap_free_mb); fm=${fm:-0}
  if [ "$paused" -eq 0 ] && [ "$fm" -lt "$STOP_BELOW_MB" ]; then
    kill -STOP "$DPID" 2>/dev/null; paused=1; log "PAUSE (swap free ${fm}M < ${STOP_BELOW_MB}M)"
  elif [ "$paused" -eq 1 ] && [ "$fm" -gt "$CONT_ABOVE_MB" ]; then
    kill -CONT "$DPID" 2>/dev/null; paused=0; log "RESUME (swap free ${fm}M > ${CONT_ABOVE_MB}M)"
  fi
  sleep 5
done
log "driver exited. recover rows=$(wc -l < "$REC" 2>/dev/null|tr -d ' ')"

# completeness-guarded remerge (won't truncate final or falsely signal done)
REC_ROWS=$(wc -l < "$REC" 2>/dev/null|tr -d ' '); REC_ROWS=${REC_ROWS:-0}
EXPECT=$(( $(ls -1 "$LOCAL_IMG"|wc -l|tr -d ' ') - $(wc -l < "$GOOD"|tr -d ' ') ))
if [ "$REC_ROWS" -lt $(( EXPECT * 90 / 100 )) ]; then
  log "INCOMPLETE recover=$REC_ROWS/$EXPECT — not remerging, re-run to resume"; exit 2
fi
python3 - "$GOOD" "$REC" "$FINAL" <<'PY' 2>> "$ST/enrich-guard.log"
import json,sys
srcs,final=sys.argv[1:-1],sys.argv[-1]; by={}
for p in srcs:
    try:
        for line in open(p):
            line=line.strip()
            if line:
                try: o=json.loads(line)
                except: continue
                s=o.get("sku")
                if s: by[s]=o
    except FileNotFoundError: pass
ok=sum(1 for o in by.values() if not o.get("error") and o.get("usable") is not False)
open(final,"w").write("\n".join(json.dumps(by[s]) for s in sorted(by))+"\n")
print(f"[remerge] final={len(by)} ok={ok}")
PY
log "REMERGE done final=$(wc -l < "$FINAL"|tr -d ' ')"; touch "$ST/GUARDED-DONE"; log "ALL DONE"