[object Object]

← back to Model Arena

Add silent-failure alerting wrapper to daily-challenge launchd job

53e7842837437f63680a88e93390695fba79d78b · 2026-07-27 14:42:49 -0700 · Steve Abrams

Wraps the 7:15am daily battle: persistent log (not /tmp) + CNCP parking-lot
card + best-effort George email on any nonzero exit or missing fresh daily-log
entry. Also un-silences the fs.appendFile catch. Closes the silent-death gap
from the /yolo contrarian gate. $0 (local only). TK-00107.

Files touched

Diff

commit 53e7842837437f63680a88e93390695fba79d78b
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Mon Jul 27 14:42:49 2026 -0700

    Add silent-failure alerting wrapper to daily-challenge launchd job
    
    Wraps the 7:15am daily battle: persistent log (not /tmp) + CNCP parking-lot
    card + best-effort George email on any nonzero exit or missing fresh daily-log
    entry. Also un-silences the fs.appendFile catch. Closes the silent-death gap
    from the /yolo contrarian gate. $0 (local only). TK-00107.
---
 scripts/com.steve.model-arena-daily.plist |  8 ++---
 scripts/daily-challenge-wrapped.sh        | 52 +++++++++++++++++++++++++++++++
 scripts/daily-challenge.js                |  2 +-
 3 files changed, 57 insertions(+), 5 deletions(-)

diff --git a/scripts/com.steve.model-arena-daily.plist b/scripts/com.steve.model-arena-daily.plist
index 04adc24..1289e8e 100644
--- a/scripts/com.steve.model-arena-daily.plist
+++ b/scripts/com.steve.model-arena-daily.plist
@@ -4,11 +4,11 @@
   <key>Label</key><string>com.steve.model-arena-daily</string>
   <key>ProgramArguments</key>
   <array>
-    <string>/opt/homebrew/bin/node</string>
-    <string>/Users/macstudio3/Projects/model-arena/scripts/daily-challenge.js</string>
+    <string>/bin/bash</string>
+    <string>/Users/macstudio3/Projects/model-arena/scripts/daily-challenge-wrapped.sh</string>
   </array>
   <key>StartCalendarInterval</key><dict><key>Hour</key><integer>7</integer><key>Minute</key><integer>15</integer></dict>
-  <key>StandardOutPath</key><string>/tmp/model-arena-daily.log</string>
-  <key>StandardErrorPath</key><string>/tmp/model-arena-daily.err</string>
+  <key>StandardOutPath</key><string>/Users/macstudio3/Projects/model-arena/yolo/daily-wrapper.log</string>
+  <key>StandardErrorPath</key><string>/Users/macstudio3/Projects/model-arena/yolo/daily-wrapper.log</string>
   <key>RunAtLoad</key><false/>
 </dict></plist>
diff --git a/scripts/daily-challenge-wrapped.sh b/scripts/daily-challenge-wrapped.sh
new file mode 100755
index 0000000..e77ec03
--- /dev/null
+++ b/scripts/daily-challenge-wrapped.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+# daily-challenge-wrapped.sh — runs the Model Arena daily battle and ALERTS on failure.
+# Wired 2026-07-27 (TK-00107) per approved
+# ~/.claude/yolo-queue/pending-approval/model-arena-daily-canary-wrapper.md
+#
+# Closes the silent-death gap the /yolo contrarian gate found: the raw job wrote
+# proof-of-life to /tmp (nobody reads, wiped on reboot) and swallowed append
+# errors, so a 7:15am failure lost a day of leaderboard data with zero signal.
+#
+# Alert channels chosen because they work headless from launchd (no MCP tools):
+#   1. persistent log file (NOT /tmp)         — audit trail that survives reboot
+#   2. CNCP parking-lot card (curl :3333)     — always-visible on Steve's board
+#   3. George email (best-effort)             — matches canary doctrine
+# Alerts fire on a nonzero exit OR when no FRESH daily-log line was appended
+# (catches the silent fs.appendFile failure class too). $0 — no metered calls.
+set -uo pipefail
+
+DIR="$HOME/Projects/model-arena"
+NODE=/opt/homebrew/bin/node
+DAYLOG="$DIR/yolo/daily-log.jsonl"
+WLOG="$DIR/yolo/daily-wrapper.log"     # persistent, survives reboot
+mkdir -p "$DIR/yolo"
+TS="$(/bin/date -u +%Y-%m-%dT%H:%M:%SZ)"
+
+# snapshot the last judged line BEFORE the run, to detect a fresh append after
+BEFORE="$(/usr/bin/tail -1 "$DAYLOG" 2>/dev/null || true)"
+
+"$NODE" "$DIR/scripts/daily-challenge.js" >>"$WLOG" 2>&1
+CODE=$?
+
+AFTER="$(/usr/bin/tail -1 "$DAYLOG" 2>/dev/null || true)"
+FRESH=NO; [ -n "$AFTER" ] && [ "$AFTER" != "$BEFORE" ] && FRESH=yes
+
+if [ "$CODE" -eq 0 ] && [ "$FRESH" = "yes" ]; then
+  echo "$TS OK exit=0 logged: $AFTER" >>"$WLOG"
+  exit 0
+fi
+
+# --- FAILURE: nonzero exit OR no fresh judged entry (silent-append guard) ---
+MSG="Model Arena daily battle FAILED $TS (exit=$CODE, freshLog=$FRESH). Tail: $WLOG"
+echo "$TS ALERT $MSG" >>"$WLOG"
+
+# 1) CNCP parking-lot card (note is a JSON body value, not an HTTP header — safe for unicode)
+/usr/bin/curl -s -m 10 -X POST http://127.0.0.1:3333/api/parking-lot \
+  -H 'Content-Type: application/json' \
+  --data-binary "$(/usr/bin/printf '{"note":"[ALERT] %s"}' "$MSG")" >/dev/null 2>&1 || true
+
+# 2) George email (best-effort; account auth baked into the CLI)
+"$HOME/bin/george" send to=info@designerwallcoverings.com \
+  subject="[ALERT] Model Arena daily job failed" body="$MSG" >/dev/null 2>&1 || true
+
+exit "$CODE"
diff --git a/scripts/daily-challenge.js b/scripts/daily-challenge.js
index df355c5..25e31d4 100644
--- a/scripts/daily-challenge.js
+++ b/scripts/daily-challenge.js
@@ -50,7 +50,7 @@ const sleep = ms => new Promise(r => setTimeout(r, ms));
     if (settled && !cur.judging && cur.judged_at) {
       const done = cur.runs.filter(r => r.status === 'done').length;
       const entry = { ts: new Date().toISOString(), id: c.id, title: pick.t, done, aiPick: cur.aiPick };
-      try { fs.appendFileSync(LOG, JSON.stringify(entry) + '\n'); } catch {}
+      try { fs.appendFileSync(LOG, JSON.stringify(entry) + '\n'); } catch (e) { console.error('LOG APPEND FAILED', e); }
       console.log('DONE', JSON.stringify(entry));
       return;
     }

← f109d9b auto-save: 2026-07-25T18:37:55 (19 files) — data/challenges.  ·  back to Model Arena  ·  auto-save: 2026-07-27T14:51:27 (3 files) — data/challenges.j 88eab1f →