`scripts/sent-stamp-core.mjs`

220 lines
// Pure decision logic for watch-sent-stamp.mjs (TK-12320). No FileMaker, no network, no fs —
// every side effect is injected so test/sent-stamp-retry.test.js can drive it with fixtures.

const pad = n => String(n).padStart(2, '0');
export const fmt = d => `${pad(d.getMonth() + 1)}/${pad(d.getDate())}/${d.getFullYear()}`;

// Memos a follow-up letter covers: ordered minAge..maxAge days before `asOf`.
export function chaseWindow(asOf, minAgeDays, maxAgeDays) {
  const hi = new Date(asOf); hi.setDate(hi.getDate() - minAgeDays);
  const lo = new Date(asOf); lo.setDate(lo.getDate() - maxAgeDays);
  return `${fmt(lo)}...${fmt(hi)}`;
}

// The chase date is the day the letter LEFT info@, not the day this poller noticed it — a retried
// send must never be stamped with a later date than the vendor actually received the letter.
export function sendDateOf(msg, now) {
  const ms = Number(msg?.internalDate);
  if (Number.isFinite(ms) && ms > 0) return { date: new Date(ms), source: 'internalDate' };
  const hdr = msg?.date ? new Date(msg.date) : null;
  if (hdr && !Number.isNaN(hdr.getTime())) return { date: hdr, source: 'date-header' };
  return { date: new Date(now), source: 'fallback-today' };
}

// Only a send that actually stamped something, with no failed commits, is finished. A skip
// (no vid mapping yet) or stamped:0 (mapping was wrong / records not visible yet) must be retried,
// otherwise a send that landed before its mapping existed is poisoned forever and re-chased.
// `covered` (letter items already stamped by an earlier send) deliberately does NOT finish a send:
// a sibling item in the same letter may not be visible in FileMaker yet, and retries are cheap.
export function isTerminal(entry) {
  if (!entry || entry.skip || Number(entry.failed) > 0) return false;
  return Number(entry.stamped) >= 1;
}

export function shouldProcess(entry, { now, retrySecs, dry }) {
  if (!entry) return { go: true, why: 'new' };
  if (isTerminal(entry)) return { go: false, why: 'terminal' };
  if (dry) return { go: true, why: 'retry' };
  const last = Date.parse(entry.last_try || entry.at || '') || 0;
  if (now - last < retrySecs * 1000) return { go: false, why: 'backoff' };
  return { go: true, why: 'retry' };
}

// Keep history instead of overwriting: attempts, first/last try, and the prior terminal-ish outcome.
export function mergeEntry(prev, outcome, now) {
  const iso = new Date(now).toISOString();
  const attempts = (Number(prev?.attempts) || (prev ? 1 : 0)) + 1;
  const first_try = prev?.first_try || prev?.at || iso;
  const next = { ...outcome, attempts, first_try, last_try: iso };
  if (prev && (prev.skip || Number(prev.stamped) === 0) && !outcome.skip && outcome.stamped >= 1) {
    next.recovered_from = prev.skip ? `skip: ${prev.skip}` : 'stamped:0';
  }
  return next;
}

// email → candidate vids. fleet.json alone is not a complete bridge (it only lists vendors with
// CURRENT outstanding items — TK-11255), so a vendor like Phillip Jeffries whose contact exists but
// who had dropped out of fleet.json was unmappable. Union in the vid(s) derivable from each contact
// slug (the inverse of lib/slug-aliases.cjs slugForVid). Additive only; body-Mfr# scoping in the
// stamper keeps a wrong candidate from ever stamping anything.
export function buildEmailVid(fleetVendors, contacts, { slugAliases = {}, slugForVid } = {}) {
  const slugVids = {};
  const add = (slug, vid) => { const v = String(vid || '').toUpperCase().trim(); if (slug && v) (slugVids[slug] = slugVids[slug] || new Set()).add(v); };
  for (const v of fleetVendors || []) add(v.slug, v.vid);
  for (const [vid, slug] of Object.entries(slugAliases)) add(slug, vid);
  if (slugForVid) {
    for (const slug of Object.keys(contacts || {})) {
      const vid = slug.toUpperCase().replace(/-/g, ' ');
      if (slugForVid(vid) === slug) add(slug, vid);
    }
  }
  const map = {};
  for (const [slug, c] of Object.entries(contacts || {})) {
    const vids = slugVids[slug]; if (!vids || !c) continue;
    // stamp_aliases: extra recipient addresses recognised as this vendor for STAMPING ONLY — send
    // routing never reads this field, so adding one cannot change who a letter goes to (TK-12351).
    for (const field of [c.sample_email, c.main_email, ...(Array.isArray(c.stamp_aliases) ? c.stamp_aliases : [])]) {
      for (const one of String(field || '').toLowerCase().split(/[,;]\s*/)) {
        const t = one.trim(); if (!t) continue;
        const set = (map[t] = map[t] || new Set()); for (const v of vids) set.add(v);
      }
    }
  }
  return map;
}

export function vidsForRecipient(emailVid, to) {
  const out = new Set();
  for (const e of (String(to || '').toLowerCase().match(/[\w.+-]+@[\w.-]+\.\w+/g) || [])) {
    if (emailVid[e]) for (const v of emailVid[e]) out.add(v);
  }
  return [...out];
}

// One pass over the sent follow-ups. deps: { emailVid, getBody(id), stamp(vids, body, win, dateStr) → {stamped:[], failed:n} }.
export async function runPass({ msgs, ledger, now, dry, retrySecs, minAgeDays, maxAgeDays, deps, log = () => {} }) {
  const seen = { ...ledger };
  const unmapped = new Set();
  const results = [];
  let acted = 0, changed = false;
  // Oldest send first: the FIRST letter that listed a memo owns its chase date; later resends then
  // find the record already stamped instead of overwriting the slot with a later date.
  const ordered = msgs.filter(m => m && m.id && !m.error)   // metadata fetch failed — try again next pass
    .sort((a, b) => sendDateOf(a, now).date - sendDateOf(b, now).date);
  const decision = new Map(ordered.map(m => [m.id, shouldProcess(seen[m.id], { now, retrySecs, dry })]));
  // An older non-terminal send in backoff must not let a newer send to the same VENDOR take the
  // chase date first: if any newer send is due, pull every older unfinished one forward with it.
  // TK-12351: "same vendor" = any shared resolved vid, not the literal to: address — one vendor can
  // be written to at two addresses (Lincrusta sample_email vs main_email). Address set is the
  // fallback only when the recipient resolves to no vid.
  const ownerKeys = m => {
    const vids = vidsForRecipient(deps.emailVid, m.to);
    if (vids.length) return vids.map(v => `vid:${v}`);
    return [`to:${(String(m.to || '').toLowerCase().match(/[\w.+-]+@[\w.-]+\.\w+/g) || []).sort().join(',')}`];
  };
  const dueOwners = new Set(ordered.filter(m => decision.get(m.id).go).flatMap(ownerKeys));
  for (const m of ordered) {
    const d = decision.get(m.id);
    if (!d.go && d.why === 'backoff' && ownerKeys(m).some(k => dueOwners.has(k))) decision.set(m.id, { go: true, why: 'retry-ahead-of-newer' });
  }
  for (const m of ordered) {
    const prev = seen[m.id];
    const d = decision.get(m.id);
    if (!d.go) continue;
    const vids = vidsForRecipient(deps.emailVid, m.to);
    const sd = sendDateOf(m, now);
    const sentOn = fmt(sd.date);
    if (!vids.length) {
      unmapped.add(String(m.to || '(no to)'));
      // sent_on kept on skips too, so the dead-letter heartbeat can age an entry after it leaves the search.
      if (!dry) { seen[m.id] = mergeEntry(prev, { skip: 'no vid for recipient', to: m.to || '', subject: m.subject || '', sent_on: sentOn, sent_on_source: sd.source }, now); changed = true; }
      results.push({ id: m.id, why: d.why, skip: 'no vid for recipient', to: m.to || '' });
      continue;
    }
    if (sd.source === 'fallback-today') log(`  ⚠ ${m.id}: no internalDate/Date header — stamping today ${sentOn}`);
    const win = chaseWindow(sd.date, minAgeDays, maxAgeDays);
    const body = await deps.getBody(m.id);
    if (body == null) { results.push({ id: m.id, why: d.why, bodyFailed: true, to: m.to || '' }); continue; }   // not an outcome — retried next pass
    const r = await deps.stamp(vids, body, win, sentOn);
    const stamped = r.stamped || [], failed = Number(r.failed) || 0, covered = Number(r.covered) || 0;
    if (!dry) {
      seen[m.id] = mergeEntry(prev, { vids, stamped: stamped.length, covered, failed, patterns: stamped, to: m.to || '', sent_on: sentOn, sent_on_source: sd.source, at: new Date(now).toISOString() }, now);
      changed = true;
    }
    acted += stamped.length;
    results.push({ id: m.id, why: d.why, vids, stamped, covered, failed, sentOn, to: m.to || '' });
  }
  return { ledger: seen, changed, acted, unmapped: [...unmapped].sort(), results };
}

// ── Dead-letter heartbeat (TK-12351) ────────────────────────────────────────────────────────────
// A send is only retried while the George search (newer_than:SEARCH_DAYS) still returns it. A
// non-terminal entry that ages out is dropped silently — so this reports, for the fleet-health
// rollup, which unresolved sends are about to fall off (or just did), and refuses PASS whenever the
// pass could not see the whole population. Three states, never two: an unmeasured pass is WARN.
const DAY = 86400e3;
const parseSentOn = s => { const m = /^(\d{2})\/(\d{2})\/(\d{4})$/.exec(String(s || '')); return m ? new Date(+m[3], +m[1] - 1, +m[2]).getTime() : NaN; };
const entryState = e => !e ? 'unprocessed' : isTerminal(e) ? 'terminal' : e.skip ? 'skip' : Number(e.failed) > 0 ? 'failed' : 'stamped:0';

// graceMs: a partial pass (Gmail quota 500s hit this watcher several times an hour) is WARN only when no
// FULLY-measured pass has completed within graceMs — the loop re-measures every 30s, so a lone transient
// miss inside the grace is not a finding, while a persistent one (no full measure for 30 min) is.
// Grace covers only the George/Gmail-side misses (search, metadata, body). FileMaker errors, an empty
// population and an unreadable ledger get none: those are not the known-transient quota class.
export function buildHeartbeat({ ledger = {}, msgs = [], results = [], now, searchDays, nearDays = 5, recentDays = 7, searchErr = null, ledgerErr = null, dry = false, lastFullMeasureAt = null, graceMs = 30 * 60e3 }) {
  const good = msgs.filter(m => m && m.id && !m.error);
  const metaFailed = msgs.filter(m => m && m.error).length;
  const seenIds = new Set(good.map(m => m.id));
  const pop = { sends_seen: good.length, terminal: 0, non_terminal: 0, unprocessed: 0, ledger_entries: Object.keys(ledger).length };
  const near_expiry = [];
  for (const m of good) {
    const e = ledger[m.id], state = entryState(e);
    if (state === 'terminal') { pop.terminal++; continue; }
    pop.non_terminal++; if (state === 'unprocessed') pop.unprocessed++;
    const sd = sendDateOf(m, now);
    const daysLeft = searchDays - (now - sd.date.getTime()) / DAY;
    if (daysLeft <= nearDays) near_expiry.push({ id: m.id, to: m.to || e?.to || '', vids: e?.vids || [], sent_on: fmt(sd.date), days_left: +daysLeft.toFixed(1), state, attempts: Number(e?.attempts) || (e ? 1 : 0) });
  }
  const not_measured = [], hard = [];
  if (searchErr) not_measured.push(`George search: ${searchErr}`);
  if (metaFailed) not_measured.push(`${metaFailed} message metadata fetch(es) failed`);
  const bodyFailed = results.filter(r => r.bodyFailed).length;
  if (bodyFailed) not_measured.push(`${bodyFailed} letter body fetch(es) failed`);
  const fmFailed = results.filter(r => Number(r.failed) > 0).length;
  if (fmFailed) hard.push(`${fmFailed} send(s) hit a FileMaker lookup/commit error`);
  if (ledgerErr) hard.push(`stamp-ledger unreadable: ${ledgerErr}`);
  if (!searchErr && good.length === 0) hard.push(`search returned 0 sends in ${searchDays}d — empty population, query or mailbox suspect`);
  const fullMeasure = !not_measured.length && !hard.length;
  const lastFull = fullMeasure ? now : lastFullMeasureAt;
  const withinGrace = lastFull != null && now - lastFull <= graceMs;
  not_measured.push(...hard);
  // Just-expired: only knowable when the search was complete (else "not returned" means nothing).
  const recently_expired = [];
  if (!searchErr && !metaFailed) {
    for (const [id, e] of Object.entries(ledger)) {
      if (seenIds.has(id) || isTerminal(e)) continue;
      const t = Number.isFinite(parseSentOn(e.sent_on)) ? parseSentOn(e.sent_on) : Date.parse(e.first_try || e.at || '');
      if (!Number.isFinite(t)) continue;                   // legacy undated entry — can't age it
      const age = (now - t) / DAY;
      if (age > searchDays && age <= searchDays + recentDays) recently_expired.push({ id, to: e.to || '', vids: e.vids || [], sent_on: e.sent_on || fmt(new Date(t)), state: entryState(e), attempts: Number(e.attempts) || 1 });
    }
  }
  const unmeasuredWarn = hard.length > 0 || (!fullMeasure && !withinGrace);
  const verdict = (unmeasuredWarn || near_expiry.length || recently_expired.length) ? 'WARN' : 'PASS';
  const bits = [`${pop.terminal}/${pop.sends_seen} sends in ${searchDays}d resolved`];
  if (near_expiry.length) bits.push(`${near_expiry.length} unresolved within ${nearDays}d of aging out`);
  if (recently_expired.length) bits.push(`${recently_expired.length} aged out unresolved in last ${recentDays}d`);
  if (not_measured.length) bits.push(`${unmeasuredWarn ? 'NOT MEASURED' : 'partial pass (full measure ' + Math.round((now - lastFull) / 1000) + 's ago)'}: ${not_measured.join('; ')}`);
  return {
    skill: 'sample-followup', source: 'watch-sent-stamp', generated_at: new Date(now).toISOString(), dry,
    verdict, status: verdict, summary: bits.join(' · '),
    full_measure: fullMeasure, last_full_measure_at: lastFull == null ? null : new Date(lastFull).toISOString(),
    search_days: searchDays, near_days: nearDays, population: pop, near_expiry, recently_expired, not_measured,
  };
}

export function failHeartbeat(err, now) {
  return { skill: 'sample-followup', source: 'watch-sent-stamp', generated_at: new Date(now).toISOString(), dry: false,
    verdict: 'FAIL', status: 'FAIL', summary: `pass threw: ${err}`, error: String(err) };
}