`scripts/run-mockups-overnight.sh`

82 lines
#!/bin/bash
# run-mockups-overnight.sh — bulk regenerate 3 variants per org, overnight.
#
# Wraps generate-mockups-v3.js with sleep-prevention, progress logging, and
# graceful resume. Default 200 orgs × 3 variants = 600 mockups. At ~130s per
# generation, that's ~22 hours single-threaded. With LIMIT=100 it's ~11 hours.
# Adjust LIMIT to fit the Mac's awake window.
#
# Usage:
#   nohup bash scripts/run-mockups-overnight.sh > logs/overnight-$(date +%Y%m%d).log 2>&1 &
#   disown
#
# Or with custom limit:
#   LIMIT=50 nohup bash scripts/run-mockups-overnight.sh ...
#
# Existing mockups are skipped unless FORCE=1.

set -euo pipefail

cd "$(dirname "$0")/.."

LIMIT="${LIMIT:-100}"
VARIANTS="${VARIANTS:-a,b,c}"
FORCE="${FORCE:-0}"
# Default Ollama target = Mac Studio 1 (192.168.1.133). Steve directive
# 2026-05-02 after running v3 + v5 + codex on Mac Studio 2 simultaneously
# froze the box (load avg 30+, 26GB VRAM swapping). Mac Studio 2 is for
# interactive work; bulk LLM goes to MS1. Override with OLLAMA_BASE=... if
# MS1 is down or you specifically want local.
OLLAMA_BASE="${OLLAMA_BASE:-http://192.168.1.133:11434}"
TS=$(date +%Y%m%d-%H%M)
LOG="logs/overnight-${TS}.log"

mkdir -p logs

echo "=== overnight mockup batch — start $(date) ==="
echo "    LIMIT=$LIMIT  VARIANTS=$VARIANTS  FORCE=$FORCE  OLLAMA_BASE=$OLLAMA_BASE"
echo "    log: $LOG"

# Prevent Mac sleep while we run; release on exit.
caffeinate_pid=""
if command -v caffeinate >/dev/null 2>&1; then
  caffeinate -i -s &
  caffeinate_pid=$!
  echo "    caffeinate pid=$caffeinate_pid (preventing sleep)"
fi
trap '[ -n "$caffeinate_pid" ] && kill "$caffeinate_pid" 2>/dev/null || true' EXIT

# Run the generator. Output streams to stdout and to LOG via tee.
# OLLAMA_BASE forwarded so generate-mockups-v3.js hits MS1 by default.
LIMIT="$LIMIT" VARIANTS="$VARIANTS" FORCE="$FORCE" OLLAMA_BASE="$OLLAMA_BASE" \
  node scripts/generate-mockups-v3.js 2>&1 | tee "$LOG"

echo "=== overnight mockup batch — done $(date) ==="
echo "Generated files: $(ls data/mockups/*-?.html 2>/dev/null | wc -l | tr -d ' ')"

# ─── Post-run failure-rate alert (>10% via George) ───
ok=$(grep -c "kb$" "$LOG" 2>/dev/null || echo 0)
fail=$(grep -c " FAIL$" "$LOG" 2>/dev/null || echo 0)
total=$(( ok + fail ))
if [ "$total" -gt 0 ]; then
  pct=$(( fail * 100 / total ))
  echo "Final: $ok ok, $fail fail, ${pct}% failure rate"
  if [ "$pct" -gt 10 ]; then
    body="<p>Mockup overnight batch finished with <b>${pct}%</b> failure rate.</p>"
    body+="<p>OK: $ok · FAIL: $fail · TOTAL: $total</p>"
    body+="<p>Last 20 FAIL lines:</p><pre>$(grep ' FAIL$' "$LOG" | tail -20 | sed 's/[<>&]/_/g')</pre>"
    body+="<p>Log: <code>$LOG</code></p>"
    payload=$(/usr/bin/python3 -c "
import json, sys
print(json.dumps({
  'to': 'steveabramsdesigns@gmail.com',
  'subject': 'pd-mockups overnight: ${pct}% failure rate',
  'body': '''$body'''
}))
")
    curl -s -m 10 -u admin:DWSecure2024! -X POST 'http://localhost:9850/api/send' \
      -H 'Content-Type: application/json' -d "$payload" > /dev/null || \
      echo "WARN: failed to send George alert"
  fi
fi