`src/jobs/magazine_watchdog.ts`

77 lines
/**
 * Watchdog for the magazine generator. Runs every N min, checks whether:
 *   1. Any generate_features.ts process is alive
 *   2. magazine_features count grew in the last 10 min
 * If both fail, kicks off a fresh `npm run magazine:gen 200` in nohup.
 *
 *   $ npx tsx src/jobs/magazine_watchdog.ts
 */
import 'dotenv/config';
import { spawn, execSync } from 'node:child_process';
import { pool, query } from '../../db/pool.ts';
import { existsSync, mkdirSync, appendFileSync } from 'node:fs';
import { homedir } from 'node:os';
import path from 'node:path';

const PROJECT = path.join(homedir(), 'Projects', 'ventura-corridor');
const LOG_DIR = path.join(PROJECT, 'logs');
const LOG_FILE = path.join(LOG_DIR, 'magazine-watchdog.log');
const STALL_MIN = 10;

function log(msg: string) {
  if (!existsSync(LOG_DIR)) mkdirSync(LOG_DIR, { recursive: true });
  const line = `[${new Date().toISOString()}] ${msg}\n`;
  appendFileSync(LOG_FILE, line);
  process.stdout.write(line);
}

async function main() {
  // 1. Is there a live generator process?
  let alive = false;
  try {
    const out = execSync(`pgrep -af 'generate_features\\.ts' || true`, { encoding: 'utf8' });
    alive = out.trim().length > 0;
  } catch {}

  // 2. Has count grown in the last STALL_MIN min?
  const r = await query(`
    SELECT
      count(*)                                                                AS total,
      count(*) FILTER (WHERE generated_at > NOW() - INTERVAL '${STALL_MIN} minutes') AS recent
    FROM magazine_features
  `);
  const total = Number(r.rows[0].total);
  const recent = Number(r.rows[0].recent);

  log(`alive=${alive} total=${total} recent_${STALL_MIN}min=${recent}`);

  if (alive && recent > 0) {
    log(`OK · gen healthy`);
    await pool.end();
    return;
  }
  if (alive && recent === 0) {
    log(`STALL · gen process alive but no rows in ${STALL_MIN} min — killing + restarting`);
    try { execSync(`pkill -9 -f 'generate_features\\.ts' || true`); } catch {}
    await new Promise(r => setTimeout(r, 2000));
  } else if (!alive) {
    log(`DOWN · no gen process — starting`);
  }

  // Spawn fresh gen, detached
  const child = spawn('bash', ['-lc',
    `cd "${PROJECT}" && nohup npx tsx src/jobs/generate_features.ts 200 >> logs/magazine-gen-overnight.log 2>&1 &`
  ], { detached: true, stdio: 'ignore' });
  child.unref();
  await new Promise(r => setTimeout(r, 3000));
  let newPid = '';
  try { newPid = execSync(`pgrep -f 'generate_features\\.ts 200' | head -1`, { encoding: 'utf8' }).trim(); } catch {}
  log(`RESTART · started new gen pid=${newPid}`);
  await pool.end();
}

main().catch(e => {
  log(`fatal: ${e.message}`);
  process.exit(1);
});