`publish-snapshot.js`

124 lines
#!/usr/bin/env node
// publish-snapshot.js  (runs on Mac2 — the source of truth)
// Bakes a COMPLETE snapshot of the doing board (tickets + already-computed
// plain-word summaries) from the live local server, then rsyncs just that one
// small JSON up to Kamatera. Kamatera serves it verbatim — no ticket log, no
// Ollama needed there. Run every few minutes via launchd.

const http = require('http');
const fs   = require('fs');
const path = require('path');
const { execFileSync } = require('child_process');

const LOCAL  = 'http://127.0.0.1:9790/api/doing';
const USER   = process.env.BASIC_USER || 'admin';
const PASS   = process.env.BASIC_PASS || 'DW2024!';
const SNAP   = path.join(__dirname, 'data', 'snapshot.json');
const REMOTE = 'root@45.61.58.125:/root/Projects/doing-viewer/data/snapshot.json';

// Connection-level errors that mean the SOURCE server (:9790) is simply not up
// right now — a self-healing condition (keep-alive + dw-uptime-probe watch that
// process; a LONG outage makes the Kamatera snapshot go stale, which
// cron-fire-canary catches via artifact-freshness). These must NOT be conflated
// with a real publish failure, so we tag them and exit 0 on this tick instead of
// flapping the canary with exit=1 (TK-11446).
const SOURCE_DOWN_CODES = new Set(['ECONNREFUSED', 'ECONNRESET', 'ENOTFOUND', 'EHOSTUNREACH']);

function getLocal() {
  return new Promise((resolve, reject) => {
    const auth = 'Basic ' + Buffer.from(`${USER}:${PASS}`).toString('base64');
    // 30s (was 20s): the local server computes plain-word summaries for ~56 doing
    // tickets on the fly, which under load occasionally crept past 20s and threw
    // 'timeout', crashing an otherwise-healthy run (TK-11446 flapping).
    http.get(LOCAL, { headers: { Authorization: auth }, timeout: 30000 }, (res) => {
      let d = ''; res.on('data', c => d += c);
      res.on('end', () => { try { resolve(JSON.parse(d)); } catch (e) { reject(e); } });
    }).on('error', (err) => {
      // Mark "source process is down" distinctly from "source up but publish broke".
      if (err && SOURCE_DOWN_CODES.has(err.code)) err.sourceDown = true;
      reject(err);
    }).on('timeout', function () { this.destroy(); reject(new Error('timeout')); });
    // NOTE: a 30s timeout is NOT treated as sourceDown — it means the server is
    // reachable but the summarizer stalled, a genuine issue worth an exit=1 warn.
  });
}

const sleep = ms => new Promise(r => setTimeout(r, ms));

// Track how long the source has been continuously unreachable, ACROSS launchd
// runs (this script runs once per 180s fire — no in-memory state survives).
// A brief blip (e.g. a pm2 restart) self-heals and should NOT flap the canary,
// but a SUSTAINED outage must stay visible: cron-fire-canary flags this job as
// artifact_weak (its publish.log refreshes every tick regardless of outcome),
// so the launchd EXIT CODE is the only signal it can trust here. We therefore
// exit 0 only while the outage is inside the self-heal grace window, and exit 1
// once it is sustained — so a real, prolonged source-down is genuinely reported
// (TK-11431: a failed input must not be silently reported as success).
const DOWN_STATE = path.join(__dirname, 'data', 'source-down-since.json');
const SOURCE_DOWN_GRACE_MS = 10 * 60 * 1000; // ~3-4 ticks: tolerate a transient restart

function clearDownState() { try { fs.unlinkSync(DOWN_STATE); } catch (_) {} }
function recordDownAndAgeMs() {
  let since = null;
  try { since = JSON.parse(fs.readFileSync(DOWN_STATE, 'utf8')).since; } catch (_) {}
  if (!since) { since = Date.now(); try { fs.writeFileSync(DOWN_STATE, JSON.stringify({ since })); } catch (_) {} }
  return Date.now() - since;
}

async function publishOnce() {
  const data = await getLocal();
  const snap = {
    syncedAt: new Date().toISOString(),
    source: 'mac2',
    count: data.count,
    items: data.items.map(i => ({          // strip nothing — summaries baked in
      id: i.id, title: i.title, project: i.project,
      agent: i.agent, doingText: i.doingText, doingTs: i.doingTs,
      ageH: i.ageH, eli12: i.eli12 || null, diag: i.diag || null,
    })),
  };
  fs.writeFileSync(SNAP, JSON.stringify(snap));
  // rsync the single file (atomic on the remote via --inplace off / temp+rename default)
  execFileSync('rsync', ['-az', '--timeout=25', SNAP, REMOTE], { stdio: 'inherit' });
  const summarized = snap.items.filter(i => i.eli12).length;
  console.log(`[${snap.syncedAt}] published ${snap.count} doing (${summarized} summarized) -> Kamatera`);
}

// Retry ONCE on a transient failure (slow local fetch or a slow Kamatera rsync)
// before deciding the exit code. A single blip no longer crash-flags the job.
// Exit-code semantics (TK-11446):
//   - SOURCE server (:9790) is simply down after a retry  -> exit 0 + WARN.
//     This is self-healing and monitored elsewhere (keep-alive restarts the pm2
//     process; a LONG outage staleness is caught by cron-fire-canary's
//     artifact-freshness check). It is NOT a failure of THIS cron job's logic,
//     so we don't flap the canary with a false exit=1 for a dependency outage.
//   - Any OTHER sustained failure (summarizer timeout, malformed JSON = source
//     up but broken, rsync-to-Kamatera failure) -> exit 1, so the canary
//     genuinely warns on a real publish-path problem.
(async () => {
  try {
    await publishOnce();
    clearDownState();               // source reachable + published -> outage (if any) is over
  } catch (e1) {
    console.error(`publish attempt 1 failed: ${e1.message} — retrying once in 5s`);
    await sleep(5000);
    try {
      await publishOnce();
      clearDownState();
    } catch (e2) {
      if (e2 && e2.sourceDown) {
        const downMs = recordDownAndAgeMs();
        const downMin = Math.round(downMs / 60000);
        if (downMs >= SOURCE_DOWN_GRACE_MS) {
          console.error(`source :9790 unreachable (${e2.code}) for ~${downMin}m (>= ${SOURCE_DOWN_GRACE_MS / 60000}m grace) — SUSTAINED outage, exit 1 so cron-fire-canary warns.`);
          process.exit(1);
        }
        console.error(`source :9790 unreachable (${e2.code}) for ~${downMin}m — within self-heal grace, skipping this tick (exit 0). keep-alive should restore :9790.`);
        process.exit(0);
      }
      console.error(`publish failed (after retry): ${e2.message}`);
      process.exit(1);
    }
  }
})();