`lib/mode-corpus.js`

113 lines
// lib/mode-corpus.js — keep the local Mode-newsletter corpus (data/mode-emails-*.js) fresh
// from Gmail via the George HTTP bridge, so the claims watch stays autonomous WITHOUT the
// (currently unset) OAuth connector. This is the "refactor the watch onto the George corpus
// path" deliverable (TK-10506).
//
// Design contract:
//   • BEST-EFFORT + NON-BREAKING. Any failure (George down, bridge 401, steve-personal token
//     invalid_grant, unexpected body shape) returns {ok:false, reason} and NEVER throws — the
//     watch always falls through to ingesting the existing corpus.
//   • $0, read-only against Gmail (no sends, no submits). Appends fetched bodies to a dedicated
//     shard data/mode-emails-george.js that the corpus ingester globs in automatically.
//   • Mode newsletter lands in the 'steve-personal' Gmail account (steveabramsdesigns@gmail.com).

const fs = require('fs');
const path = require('path');

const GEORGE_URL = (process.env.GEORGE_URL || '').replace(/\/$/, '');
const GEORGE_BASIC_AUTH = process.env.GEORGE_BASIC_AUTH || '';
const ACCOUNT = process.env.MODE_MAIL_ACCOUNT || 'steve-personal';
const FROM = 'daniel@mail.modemobile.com';
const SHARD = path.join(__dirname, '..', 'data', 'mode-emails-george.js');

function authHeaders() {
  return GEORGE_BASIC_AUTH ? { Authorization: 'Basic ' + GEORGE_BASIC_AUTH } : {};
}

async function gjson(url) {
  const res = await fetch(url, { headers: authHeaders() });
  const text = await res.text();
  let json; try { json = JSON.parse(text); } catch (_) { json = null; }
  if (!res.ok) {
    const msg = (json && (json.error || json.message)) || text.slice(0, 120);
    const err = new Error(`George ${res.status}: ${msg}`);
    err.status = res.status;
    err.invalidGrant = /invalid_grant/i.test(text);
    throw err;
  }
  return json;
}

function loadShard() {
  try { const arr = require(SHARD); return Array.isArray(arr) ? arr : []; }
  catch (_) { return []; }
}

function writeShard(list) {
  const banner = '// data/mode-emails-george.js — AUTO-MAINTAINED by lib/mode-corpus.js.\n' +
    '// Mode-newsletter bodies fetched from Gmail via the George bridge. Do not hand-edit.\n';
  fs.writeFileSync(SHARD, banner + 'module.exports = ' + JSON.stringify(list, null, 2) + ';\n');
}

// Try the known George body endpoints in order; return a plain-text body or null.
async function fetchBody(id) {
  const enc = encodeURIComponent(id);
  const acct = encodeURIComponent(ACCOUNT);
  const candidates = [
    `${GEORGE_URL}/api/message?account=${acct}&id=${enc}&format=text`,
    `${GEORGE_URL}/api/messages/${enc}?account=${acct}`,
    `${GEORGE_URL}/api/message/${enc}?account=${acct}`,
  ];
  for (const url of candidates) {
    try {
      const j = await gjson(url);
      const body = j && (j.body || j.text || j.plain || (j.message && (j.message.body || j.message.text)));
      if (body && String(body).trim()) return String(body);
    } catch (e) { if (e.invalidGrant) throw e; /* try next shape */ }
  }
  return null;
}

/**
 * Refresh the corpus shard from George. Returns {ok, added, reason}.
 * Never throws.
 */
async function refreshFromGeorge({ days = 20 } = {}) {
  if (!GEORGE_URL) return { ok: false, added: 0, reason: 'GEORGE_URL unset' };
  let list;
  try {
    const q = encodeURIComponent(`from:${FROM} newer_than:${days}d`);
    const j = await gjson(`${GEORGE_URL}/api/messages?account=${encodeURIComponent(ACCOUNT)}&maxResults=50&q=${q}`);
    list = j.messages || j || [];
    if (!Array.isArray(list)) list = [];
  } catch (e) {
    // invalid_grant => steve-personal needs a Steve-gated OAuth reconnect. Surface it clearly.
    const reason = e.invalidGrant
      ? `george account '${ACCOUNT}' invalid_grant — reconnect required (gated)`
      : `george list failed: ${e.message}`;
    return { ok: false, added: 0, reason, invalidGrant: !!e.invalidGrant };
  }

  const shard = loadShard();
  const have = new Set(shard.map(e => e.id));
  let added = 0;
  for (const m of list) {
    const id = m.id;
    if (!id || have.has(id)) continue;
    let body = m.body || m.text || null;
    if (!body) { try { body = await fetchBody(id); } catch (e) {
      if (e.invalidGrant) return { ok: false, added, reason: `invalid_grant mid-fetch — reconnect required (gated)`, invalidGrant: true };
      body = null;
    } }
    if (!body) continue;                        // can't parse without a body; skip honestly
    const date = m.internalDate ? new Date(Number(m.internalDate)).toISOString().slice(0, 10)
      : (m.date ? new Date(m.date).toISOString().slice(0, 10) : null);
    shard.push({ id, date, subject: m.subject || '', body });
    have.add(id); added++;
  }
  if (added) writeShard(shard);
  return { ok: true, added, reason: added ? `fetched ${added} new mailer(s)` : 'no new mailers' };
}

module.exports = { refreshFromGeorge, SHARD };