`poll.mjs`

176 lines
#!/usr/bin/env node
// george-token-broker / poll.mjs
// Health-watchdog for the local George Gmail agent (NOT a token re-minter — minting a
// USER refresh token needs interactive consent, so headless re-mint is impossible).
//
// Each run: probe George's /health (process liveness) + every account's /api/profile
// (refresh-token liveness), classify, write data/latest.json for the meta-watchdog,
// append data/poll.log. On failure it takes the CORRECT action per failure mode:
//   • process down / unreachable  -> WOULD restart the launchd job + alert
//   • token dead (invalid_grant)  -> WOULD alert ONLY (a restart reloads the same dead
//                                     token; this needs interactive re-consent)
//   • all ok                      -> quiet, just records state
//
// DRY_RUN (default true) logs "WOULD: ..." and changes NOTHING live. Set DRY_RUN=0 to
// actually restart / alert. Full plan + gated root-fix:
//   ~/.claude/yolo-queue/pending-approval/george-oauth-durable-fix-2026-06-29.md

import fs from 'node:fs';
import { execFile } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import path from 'node:path';

const HERE = path.dirname(fileURLToPath(import.meta.url));
const cfg = JSON.parse(fs.readFileSync(path.join(HERE, 'config.json'), 'utf8'));
const DRY_RUN = process.env.DRY_RUN !== '0'; // default true — safe
const nowISO = () => new Date().toISOString();
const authHeader = 'Basic ' + Buffer.from(cfg.auth).toString('base64');

// Pre-emptive token-age WARN config (Council 2026-06-30 #2). The Testing-mode OAuth app
// caps refresh tokens at ~7 days; we warn at WARN_DAYS so re-consent is a scheduled 30-sec
// task, not a silent mid-pipeline death. Accounts on service-account/domain-wide-delegation
// have no 7-day token, so list them in cfg.sa_backed to skip the age warn once DWD is live.
const WARN_DAYS = Number(cfg.warn_days) || 6;
const WARN_THROTTLE_MS = 24 * 3600 * 1000; // at most one age-WARN per account per day
const SA_BACKED = new Set(cfg.sa_backed || []);
const AGES_FILE = path.join(HERE, 'data', 'token-ages.json');
const AUTH_PATH = { 'steve-office': '/auth', info: '/auth/info', agentabrams: '/auth/agentabrams', 'steve-personal': '/auth/steve-personal' };

function logLine(s) {
  const line = `[${nowISO()}] ${s}`;
  console.log(line);
  try { fs.appendFileSync(path.join(HERE, 'data', 'poll.log'), line + '\n'); } catch {}
}

async function fetchJSON(url, opts = {}) {
  const ctrl = new AbortController();
  const t = setTimeout(() => ctrl.abort(), cfg.timeout_ms);
  try {
    const r = await fetch(url, { ...opts, signal: ctrl.signal });
    const text = await r.text();
    let body; try { body = JSON.parse(text); } catch { body = { _raw: text }; }
    return { ok: r.ok, status: r.status, body, text };
  } catch (e) {
    return { ok: false, status: 0, body: null, text: String(e && e.message || e) };
  } finally { clearTimeout(t); }
}

// Classify one account probe -> 'ok' | 'token_dead' | 'error'
function classify(res) {
  if (res.ok && res.body && res.body.emailAddress) return { state: 'ok', detail: res.body.emailAddress };
  const blob = (res.text || '') + JSON.stringify(res.body || {});
  if (/invalid_grant|Token has been expired or revoked|invalid_request.*refresh/i.test(blob))
    return { state: 'token_dead', detail: 'invalid_grant — refresh token expired/revoked (needs re-consent)' };
  return { state: 'error', detail: `http ${res.status}: ${blob.slice(0, 160)}` };
}

function run(cmd, args) {
  return new Promise((resolve) => {
    execFile(cmd, args, { timeout: 15000 }, (err, stdout, stderr) =>
      resolve({ err: err ? String(err.message) : null, stdout, stderr }));
  });
}

async function restartGeorge() {
  const label = cfg.launchd_label;
  const target = `gui/${process.getuid()}/${label}`;
  if (DRY_RUN) { logLine(`WOULD restart launchd job: launchctl kickstart -k ${target}`); return; }
  logLine(`RESTART launchd job: ${target}`);
  const r = await run('launchctl', ['kickstart', '-k', target]);
  logLine(`restart result: err=${r.err || 'none'}`);
}

async function alert(summary, detail) {
  if (DRY_RUN) { logLine(`WOULD alert -> "${summary}"`); return; }
  // Fallback chain that does NOT depend on George (George may be the thing down):
  // 1) CNCP parking-lot. 2) terminal-notifier if present. Never throws.
  try {
    await fetchJSON(`${cfg.cncp_base}/api/parking-lot`, {
      method: 'POST', headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ url: 'george-token-broker', note: `⚠ George: ${summary} — ${detail}` })
    });
    logLine(`alert -> CNCP parking-lot posted`);
  } catch (e) { logLine(`alert CNCP failed: ${e}`); }
}

// Pre-emptive (not a failure) — token still works but is nearing the ~7-day cap.
async function warnAge(acct, addr, ageDays, authUrl) {
  const summary = `${acct} (${addr}) token ~${ageDays.toFixed(1)}d old — expires ~${WARN_DAYS + 1}d (Testing-mode). Re-consent soon at ${authUrl}`;
  if (DRY_RUN) { logLine(`WOULD age-WARN -> "${summary}"`); return; }
  try {
    await fetchJSON(`${cfg.cncp_base}/api/parking-lot`, {
      method: 'POST', headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ url: `george-token-broker#${acct}`, note: `⏳ George pre-emptive token-age WARN: ${summary}` })
    });
    logLine(`age-WARN -> CNCP parking-lot posted (${acct})`);
  } catch (e) { logLine(`age-WARN CNCP failed: ${e}`); }
}

(async () => {
  fs.mkdirSync(path.join(HERE, 'data'), { recursive: true });
  const health = await fetchJSON(`${cfg.george_base}/health`);
  const processUp = health.ok && health.body && health.body.status === 'ok';

  const accounts = {};
  if (processUp) {
    for (const acct of cfg.accounts) {
      const res = await fetchJSON(`${cfg.george_base}/api/profile?account=${encodeURIComponent(acct)}`,
        { headers: { Authorization: authHeader } });
      accounts[acct] = classify(res);
    }
  }

  const deadTokens = Object.entries(accounts).filter(([, v]) => v.state === 'token_dead').map(([k]) => k);
  const errored = Object.entries(accounts).filter(([, v]) => v.state === 'error').map(([k]) => k);
  let overall = 'ok';
  if (!processUp) overall = 'process_down';
  else if (deadTokens.length) overall = 'token_dead';
  else if (errored.length) overall = 'degraded';

  // ── Pre-emptive token-age WARN (Council 2026-06-30 #2) ─────────────────────────
  // first_good_ts = when the CURRENT token generation was first seen healthy; reset it
  // whenever an account returns to 'ok' from dead/error/absent (a re-consent minted a
  // fresh token). Warn once a live generation crosses WARN_DAYS, throttled to 1/day.
  const prevAcct = (() => { try { return JSON.parse(fs.readFileSync(path.join(HERE, 'data', 'latest.json'), 'utf8')).accounts || {}; } catch { return {}; } })();
  let ages = (() => { try { return JSON.parse(fs.readFileSync(AGES_FILE, 'utf8')); } catch { return {}; } })();
  const nowMs = Date.now();
  const warnings = [];
  for (const [acct, v] of Object.entries(accounts)) {
    if (v.state !== 'ok' || SA_BACKED.has(acct)) continue;
    let rec = ages[acct];
    if (!rec || (prevAcct[acct] && prevAcct[acct].state) !== 'ok') rec = { first_good_ts: nowISO(), last_warn_ts: null }; // fresh generation
    const ageDays = (nowMs - Date.parse(rec.first_good_ts)) / 86400000;
    if (ageDays > WARN_DAYS && (!rec.last_warn_ts || nowMs - Date.parse(rec.last_warn_ts) > WARN_THROTTLE_MS)) {
      warnings.push({ acct, addr: v.detail, age_days: Number(ageDays.toFixed(1)), auth_path: AUTH_PATH[acct] || '/auth' });
      rec.last_warn_ts = nowISO();
    }
    ages[acct] = rec;
  }
  try { fs.writeFileSync(AGES_FILE, JSON.stringify(ages, null, 2)); } catch {}

  const state = {
    ts: nowISO(), overall, processUp,
    uptime_s: processUp ? health.body.uptime : null,
    accounts, deadTokens, errored, warnings, dry_run: DRY_RUN
  };
  fs.writeFileSync(path.join(HERE, 'data', 'latest.json'), JSON.stringify(state, null, 2));
  logLine(`overall=${overall} processUp=${processUp} dead=[${deadTokens}] err=[${errored}] ageWarn=[${warnings.map(w => w.acct).join(',')}]`);

  // Correct action per failure mode:
  if (!processUp) {
    await restartGeorge();
    await alert('process down/unreachable', health.text || 'no /health response');
  } else if (deadTokens.length) {
    // Restart will NOT help a dead refresh token — alert only.
    await alert(`refresh token dead for ${deadTokens.join(', ')}`,
      'Publish OAuth app to production + re-consent. See pending-approval/george-oauth-durable-fix-2026-06-29.md');
  } else if (errored.length) {
    await alert(`probe error for ${errored.join(', ')}`, JSON.stringify(accounts));
  }

  // Pre-emptive age warnings are independent of health — emit even when overall=ok.
  for (const w of warnings) await warnAge(w.acct, w.addr, w.age_days, `${cfg.george_base}${w.auth_path}`);

  process.exit(overall === 'ok' ? 0 : 3);
})();