Recover stale YOLO cooldown loops

7f5e98ea9c4be78b7b1f1e67f80b7e77b92d8cfe · 2026-08-30 23:16:37 -0700 · Steve Abrams
Files touched

M scripts/yolo-ticket-sync.mjs
Diff

commit 7f5e98ea9c4be78b7b1f1e67f80b7e77b92d8cfe
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Sun Aug 30 23:16:37 2026 -0700

    Recover stale YOLO cooldown loops
---
 scripts/yolo-ticket-sync.mjs | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/scripts/yolo-ticket-sync.mjs b/scripts/yolo-ticket-sync.mjs
index eaa37869..c6f34ced 100644
--- a/scripts/yolo-ticket-sync.mjs
+++ b/scripts/yolo-ticket-sync.mjs
@@ -12,6 +12,7 @@ const SSH = '/usr/bin/ssh';
 const HOST = 'root@45.61.58.125';
 const REMOTE_STATUS = `YOLO_PASS=$(pm2 jlist | jq -r '.[]|select(.name=="yolo-agent").pm2_env.YOLO_AUTH_PASS'); curl -fsS -u "admin:$YOLO_PASS" http://127.0.0.1:9670/api/status`;
 const REMOTE_START = `YOLO_PASS=$(pm2 jlist | jq -r '.[]|select(.name=="yolo-agent").pm2_env.YOLO_AUTH_PASS'); curl -fsS -u "admin:$YOLO_PASS" -X POST http://127.0.0.1:9670/api/start`;
+const REMOTE_STOP_START = `YOLO_PASS=$(pm2 jlist | jq -r '.[]|select(.name=="yolo-agent").pm2_env.YOLO_AUTH_PASS'); curl -fsS -u "admin:$YOLO_PASS" -X POST http://127.0.0.1:9670/api/stop >/dev/null; sleep 2; curl -fsS -u "admin:$YOLO_PASS" -X POST http://127.0.0.1:9670/api/start`;
 
 function ssh(command) {
   return execFileSync(SSH, ['-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', HOST, command], {
@@ -46,6 +47,17 @@ try {
     tk('challenge', `YOLO loop was stopped; monitor issued the existing /api/start recovery (${response.msg || 'started'}). No email; $0 spend.`);
   }
 
+  const lastRunMs = Date.parse(status.lastRunAt || result?.completedAt || '');
+  const cooldownMs = Math.max(5, Number(status.cooldownSec) || 600) * 1000;
+  const staleAfterMs = Math.max(30 * 60_000, cooldownMs * 2 + 5 * 60_000);
+  const stale = status.running === true && !status.currentTask && Number.isFinite(lastRunMs)
+    && Date.now() - lastRunMs > staleAfterMs;
+  if (stale && state.lastRecoveredTaskId !== result?.taskId) {
+    const response = JSON.parse(ssh(REMOTE_STOP_START));
+    tk('challenge', `YOLO loop was running but stale for ${Math.round((Date.now() - lastRunMs) / 60_000)} minutes with no current task. Monitor issued one stop/start recovery (${response.msg || 'started'}). No email; $0 spend.`);
+    state.lastRecoveredTaskId = result?.taskId || `stale-${lastRunMs}`;
+  }
+
   if (result?.taskId && result.taskId !== state.lastTaskId) {
     const summary = String(result.stdout || '').replace(/\s+/g, ' ').trim().slice(0, 700);
     const railProof = String(result.stderr || '').includes('No Shopify write')
@@ -53,6 +65,9 @@ try {
     const text = `YOLO cycle #${result.taskId} ${result.status} in ${result.duration || 'unknown duration'}. ${summary || 'No summary returned.'}${railProof} Loop running=${status.running}; mode=${status.mode}; cooldown=${status.cooldownSec}s. No email notification path; spend ceiling <=$5.`;
     tk(result.status === 'success' ? 'win' : 'challenge', text);
     state.lastTaskId = result.taskId;
+    if (state.lastRecoveredTaskId && state.lastRecoveredTaskId !== result.taskId) {
+      state.lastRecoveredTaskId = null;
+    }
   }
 
   if (state.lastRunning !== status.running) {
← 32dadef2 record Kamatera PM2 fracture prevention rollout · back to Ticket System · Verify YOLO recovery before ticket updates 36de9af9 →