`health.js`

101 lines
'use strict';
// claude-mail self-heal / alert layer.
// The poller calls recordFailure(msg) when a poll throws and recordSuccess()
// when one completes. We track CONSECUTIVE failures in a small state file and,
// once they cross a threshold, fire an alert over channels that DO NOT depend
// on the (possibly-broken) Purelymail mailbox: CNCP (local HTTP), a native
// macOS notification, and George/Gmail (separate auth) as a bonus. On recovery
// we fire a one-shot "back online" notice and reset the counter.

const fs = require('node:fs');
const path = require('node:path');
const { execFile } = require('node:child_process');

const STATE = path.join(__dirname, '.health.json');
const THRESHOLD = Number(process.env.HEALTH_FAIL_THRESHOLD || 3);      // consecutive fails before alerting (~30s @ 10s poll)
const REALERT_MS = Number(process.env.HEALTH_REALERT_MS || 30 * 60 * 1000); // re-page at most every 30 min during an outage
const CNCP = process.env.CNCP_URL || 'http://127.0.0.1:3333';
const GEORGE = process.env.GEORGE_URL || 'http://127.0.0.1:9850';
const ALERT_TO = process.env.HEALTH_ALERT_EMAIL || 'steveabramsdesigns@gmail.com';

function read() {
  try { return JSON.parse(fs.readFileSync(STATE, 'utf8')); }
  catch { return { consecutiveFailures: 0, firstFailureAt: null, lastAlertAt: null, lastError: null }; }
}
function write(s) { try { fs.writeFileSync(STATE, JSON.stringify(s, null, 2)); } catch {} }
const log = (...a) => console.log(new Date().toISOString(), '[health]', ...a);

// ---- alert channels (all best-effort, none may throw) ----------------------
function macNotify(title, msg) {
  return new Promise((res) => {
    const safe = (s) => String(s).replace(/["\\]/g, ' ').slice(0, 240);
    execFile('osascript', ['-e', `display notification "${safe(msg)}" with title "${safe(title)}" sound name "Basso"`],
      () => res());
  });
}
async function cncpAlert(title, note) {
  // Land it in the parking lot so it shows up in the command center Steve watches.
  for (const ep of ['/api/parking-lot', '/api/parkinglot', '/api/wins']) {
    try {
      const body = ep.includes('wins')
        ? { project: 'claude-mail', title, summary: note }
        : { title, note, source: 'claude-mail-health' };
      const r = await fetch(CNCP + ep, {
        method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body),
      });
      if (r.ok) { log(`CNCP alert posted via ${ep}`); return true; }
    } catch {}
  }
  return false;
}
async function georgeAlert(subject, text) {
  // George has its own (Gmail) auth, independent of the Purelymail mailbox.
  try {
    const r = await fetch(GEORGE + '/api/send', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json', 'Authorization': 'Basic ' + Buffer.from('admin:').toString('base64') },
      body: JSON.stringify({ account: 'steve-office', to: ALERT_TO, subject, body: `<pre>${text}</pre>` }),
    });
    if (r.ok) { log('George alert email sent'); return true; }
  } catch {}
  return false;
}

async function fireAlert(kind, detail) {
  const title = kind === 'down' ? '🔴 claude-mail poller is FAILING' : '🟢 claude-mail poller recovered';
  const note = kind === 'down'
    ? `The claude@agentabrams.com poller has failed ${detail.count}x in a row since ${detail.since}. Last error: ${detail.error}. The email channel is NOT reading mail. Check: cd ~/Projects/claude-mail && node --env-file=.env probe.js`
    : `claude-mail poller is back: it failed ${detail.count}x but a poll just succeeded. The channel is reading mail again.`;
  log(`FIRING ${kind} alert: ${note}`);
  await Promise.all([ macNotify(title, note), cncpAlert(title, note), georgeAlert(title, note) ]);
}

// ---- public API ------------------------------------------------------------
async function recordFailure(errMsg) {
  const s = read();
  s.consecutiveFailures += 1;
  s.lastError = String(errMsg || 'unknown');
  if (!s.firstFailureAt) s.firstFailureAt = new Date().toISOString();
  const now = Date.now();
  const due = !s.lastAlertAt || (now - new Date(s.lastAlertAt).getTime()) > REALERT_MS;
  if (s.consecutiveFailures >= THRESHOLD && due) {
    s.lastAlertAt = new Date().toISOString();
    write(s); // persist before the (async) alert so a crash mid-alert can't double-fire
    await fireAlert('down', { count: s.consecutiveFailures, since: s.firstFailureAt, error: s.lastError });
  } else {
    write(s);
  }
}
async function recordSuccess() {
  const s = read();
  if (s.consecutiveFailures >= THRESHOLD) {
    // We had paged about an outage; tell Steve it cleared.
    await fireAlert('recovered', { count: s.consecutiveFailures });
  }
  if (s.consecutiveFailures !== 0 || s.lastAlertAt) {
    write({ consecutiveFailures: 0, firstFailureAt: null, lastAlertAt: null, lastError: null });
  }
}

module.exports = { recordFailure, recordSuccess };