← back to Codex Review 2026 05 03

nightly.sh

265 lines

#!/usr/bin/env bash
# nightly.sh — overnight YOLO debate-review across the 25 Apr-30 projects.
# Hard cutoff: 7am PT Mon May 4. Per-project budget: 18 min wall-clock.
# Auto-applies tightly-scoped safe patches; queues everything else for morning.
# Halts and emails on >5 high-severity findings or any test regression.

set -uo pipefail

ROOT="$HOME/Projects/codex-review-2026-05-03"
mkdir -p "$ROOT"
LOG="$ROOT/master.log"
DIGEST="$ROOT/digest.md"
TODO="$ROOT/morning-review.md"
HALT_FLAG="$ROOT/halt.flag"

# Hard cutoff (7am PT Mon May 4 = epoch 1746363600 in PDT)
CUTOFF_EPOCH=$(date -j -f "%Y-%m-%d %H:%M:%S" "2026-05-04 07:00:00" +%s 2>/dev/null || echo 0)
PER_PROJECT_BUDGET_SEC=1080  # 18 min

# 25 projects from Apr-30 launch-all.sh
PROJECTS=(
  bankrupt-leads professional-directory lawyer-directory-builder
  site-factory the-ai-factory visual-factory secrets-manager
  Hormuzy resize-it yolo-agent Forza trademarks-copyright Ken
  VictoryStays malden-house jill-website stayclaim wholivedthere
  claimmyaddress bubbesblock Letsbegin dear-bubbe-next AgentAbrams
  Designer-Wallcoverings Norma
)

log() { echo "[$(date '+%H:%M:%S')] $*" | tee -a "$LOG" >&2; }

cat > "$DIGEST" <<EOF
# Overnight YOLO · 2026-05-03 → 2026-05-04
**Started:** $(date)
**Hard cutoff:** Mon 2026-05-04 07:00 PT
**Projects in queue:** ${#PROJECTS[@]}
**Mode:** claude-codex 8-way debate (rounds=2) → safe-patch only · risky → \`$TODO\`

---

EOF

cat > "$TODO" <<EOF
# Morning Review Queue · 2026-05-04
Findings the overnight loop did NOT auto-apply. Sorted by project. Severity: H/M/L.

EOF

START_EPOCH=$(date +%s)
TOTAL_AUTO=0
TOTAL_QUEUED=0
TOTAL_HIGHSEV=0
HALT_REASON=""

for P in "${PROJECTS[@]}"; do
  NOW=$(date +%s)
  if [ "$CUTOFF_EPOCH" -gt 0 ] && [ "$NOW" -ge "$CUTOFF_EPOCH" ]; then
    log "═══ 7am cutoff reached — stopping queue at $P"
    HALT_REASON="hard cutoff 07:00 PT"
    break
  fi
  if [ -f "$HALT_FLAG" ]; then
    log "═══ HALT flag detected — stopping at $P"
    HALT_REASON="HALT flag: $(cat "$HALT_FLAG" 2>/dev/null)"
    break
  fi

  SRC="$HOME/Projects/$P"
  OUT="$ROOT/$P"
  mkdir -p "$OUT"

  log "═══ $P start ═══"
  echo "" >> "$DIGEST"
  echo "## $P" >> "$DIGEST"
  echo "Workdir: \`$SRC\`" >> "$DIGEST"

  if [ ! -d "$SRC" ]; then
    log "  SKIP: $SRC missing"; echo "- skipped (missing)" >> "$DIGEST"; continue
  fi

  # Ensure git repo (claude-codex requires one)
  if [ ! -d "$SRC/.git" ]; then
    log "  initializing git repo (transient — for debate scope only)"
    (cd "$SRC" && git init -q && git add -A 2>/dev/null && git commit -qm "[overnight] pre-debate baseline" --allow-empty 2>/dev/null) || true
  fi

  # 1. Launch debate (8-way, rounds=2 to fit budget)
  RUN_NAME="overnight-2026-05-03-$P"
  log "  starting debate $RUN_NAME"
  ~/.claude/skills/claude-codex/scripts/start.sh \
    --name "$RUN_NAME" --workdir "$SRC" --rounds 2 --report-only \
    >> "$LOG" 2>&1 || { log "  start.sh FAILED for $P"; echo "- ERROR: start.sh failed" >> "$DIGEST"; continue; }

  RUN_DIR="$HOME/.claude/skills/claude-codex/runs/$RUN_NAME"
  PID_FILE="$RUN_DIR/loop.pid"
  if [ ! -f "$PID_FILE" ]; then
    log "  no pid file — debate failed to spawn"; echo "- ERROR: no pid" >> "$DIGEST"; continue
  fi
  DEBATE_PID=$(cat "$PID_FILE")
  log "  debate pid=$DEBATE_PID — polling (max ${PER_PROJECT_BUDGET_SEC}s)"

  WAITED=0
  while kill -0 "$DEBATE_PID" 2>/dev/null; do
    sleep 30
    WAITED=$((WAITED + 30))
    if [ $WAITED -ge $PER_PROJECT_BUDGET_SEC ]; then
      log "  TIMEOUT — killing $DEBATE_PID"
      kill -TERM "$DEBATE_PID" 2>/dev/null || true
      sleep 5
      kill -KILL "$DEBATE_PID" 2>/dev/null || true
      break
    fi
  done
  log "  debate ended after ${WAITED}s"

  # 2. Capture findings
  CROSS="$RUN_DIR/cross_exam.md"
  if [ ! -f "$CROSS" ]; then
    # Fallback to whatever round files exist
    CROSS=$(ls -t "$RUN_DIR"/round*.md 2>/dev/null | head -1)
  fi

  if [ -z "${CROSS:-}" ] || [ ! -f "$CROSS" ]; then
    log "  no findings doc produced"; echo "- (no findings)" >> "$DIGEST"; continue
  fi

  cp "$CROSS" "$OUT/findings.md"

  # Count high-severity (heuristic: lines containing "P0", "CRIT", "HIGH", or "🚨")
  HIGHCT=$(grep -ciE '(P0|CRIT|HIGH-SEV|HIGH severity|🚨)' "$CROSS" || echo 0)
  HIGHCT=$(echo "$HIGHCT" | tr -d '[:space:]')
  TOTAL_HIGHSEV=$((TOTAL_HIGHSEV + HIGHCT))

  log "  high-severity heuristic count: $HIGHCT"
  echo "- findings file: \`$OUT/findings.md\` · high-sev≈${HIGHCT}" >> "$DIGEST"

  if [ "$HIGHCT" -gt 5 ]; then
    log "  >5 high-sev — HALTING and queuing for morning (no auto-patch)"
    {
      echo ""
      echo "## $P · HALT-LEVEL FINDINGS ($HIGHCT high-sev)"
      echo "Review \`$OUT/findings.md\` first thing."
      echo ""
    } >> "$TODO"
    TOTAL_QUEUED=$((TOTAL_QUEUED + HIGHCT))
    continue
  fi

  # 3. Apply safe patches via Claude CLI subprocess (tightly scoped)
  PATCHER_LOG="$OUT/patcher.log"
  PATCHER_PROMPT="You are an overnight-mode patcher. The 8-way debate produced findings at $CROSS for the project at $SRC.

Read $CROSS. For each finding apply exactly ONE rule:

AUTO-APPLY ONLY IF the finding is one of:
  - Typo / grammar in comments, docstrings, README, log strings
  - Dead import / unused variable removal
  - Missing semicolon / formatting
  - Obvious off-by-one in test/scripts/dev-only paths (NOT in app code)
  - Leaked secret in test fixture or scratch file (rotate via secrets-manager skill if production secret)

QUEUE FOR MORNING (write to $TODO under '## $P') if it touches:
  - Logic / control flow / business rules
  - API surface / function signatures
  - Database schema / migrations
  - Configuration / env vars / dependencies / package.json
  - Anything in node_modules
  - Files serving a CNCP-listed domain (\$HOME/cncp-starter/cncp-config.json domains[])
  - Files in \$HOME/Projects/Designer-Wallcoverings (DW Shopify writes need explicit Steve auth)
  - dw_unified schema (any file containing 'dw_unified.')
  - DNS, scheduled-job changes, pm2 restarts

NEVER:
  - Restart pm2 / kill processes
  - Push to git remotes
  - Modify .env files (queue rotation request instead)
  - Touch designerwallcoverings.com or studentdebtcrisis(center).org-related code
  - Run npm install / pip install / brew install
  - Use \\\$VAR-interpolation of secrets in bash commands

When you apply a fix:
  - Commit locally with message '[overnight-yolo 2026-05-03] $P: <one-line summary>' (do NOT push)

After processing, append to $DIGEST under '## $P' a line:
  '**Patcher:** N auto-applied · M queued for morning'

Time budget: 12 minutes. If incomplete, stop cleanly and queue the rest.

Begin."

  log "  invoking patcher (12-min budget)"
  if command -v gtimeout >/dev/null 2>&1; then
    gtimeout 720 claude --print "$PATCHER_PROMPT" >> "$PATCHER_LOG" 2>&1 || log "  patcher exited (timeout/err)"
  else
    # macOS fallback: background + kill
    claude --print "$PATCHER_PROMPT" >> "$PATCHER_LOG" 2>&1 &
    PPID_=$!
    PWAITED=0
    while kill -0 "$PPID_" 2>/dev/null; do
      sleep 20; PWAITED=$((PWAITED + 20))
      if [ $PWAITED -ge 720 ]; then
        kill -TERM "$PPID_" 2>/dev/null || true; sleep 3; kill -KILL "$PPID_" 2>/dev/null || true
        log "  patcher TIMEOUT"
        break
      fi
    done
  fi
  log "  $P done"
done

END_EPOCH=$(date +%s)
ELAPSED=$((END_EPOCH - START_EPOCH))
HRS=$((ELAPSED / 3600)); MIN=$(( (ELAPSED % 3600) / 60 ))

cat >> "$DIGEST" <<EOF

---
## Run Summary
- **Wall time:** ${HRS}h ${MIN}m
- **Total high-severity findings (heuristic):** ${TOTAL_HIGHSEV}
- **Halt reason:** ${HALT_REASON:-completed full queue}
- **Per-project run dirs:** \`~/.claude/skills/claude-codex/runs/overnight-2026-05-03-*\`
- **Findings docs:** \`$ROOT/<project>/findings.md\`
- **Patcher logs:** \`$ROOT/<project>/patcher.log\`
- **Morning queue:** \`$TODO\`
EOF

log "════ run complete · ${HRS}h ${MIN}m · halt=${HALT_REASON:-none} ════"

# Email digest via George to steve-office
SUBJECT="Overnight YOLO · 2026-05-04 · ${#PROJECTS[@]} projects · ${HRS}h ${MIN}m · ${TOTAL_HIGHSEV} high-sev"
python3 - <<PYEOF
import json, urllib.request, base64
body = open("$DIGEST").read()
todo_excerpt = ""
try:
    with open("$TODO") as f:
        t = f.read()
        if len(t) > 200:
            todo_excerpt = "\n\n---\n\n## Morning Queue (full file at $TODO)\n\n" + t[:8000]
except: pass
body = body + todo_excerpt
payload = {
    "to": "steve@designerwallcoverings.com",
    "subject": "$SUBJECT",
    "body": body,
    "isHtml": False
}
req = urllib.request.Request(
    "http://localhost:9850/api/send?account=steve-office",
    data=json.dumps(payload).encode(),
    headers={
        "Content-Type": "application/json",
        "Authorization": "Basic " + base64.b64encode(b"admin:DWSecure2024!").decode()
    },
    method="POST"
)
try:
    r = urllib.request.urlopen(req, timeout=20)
    print("george send:", r.status)
except Exception as e:
    print("george send err:", e)
PYEOF
log "Done."