`mailers/cc-api/import-fm-clients.js`

148 lines
#!/usr/bin/env node
/**
 * Import the segmented, hygiene-passed FileMaker client list into a new
 * Constant Contact list. SAFE BY DEFAULT — dry-run (mock) unless BOTH
 * CC_LIVE=1 and CONFIRM=1 are set. Reads the MAILABLE segment only.
 *
 *   Dry run (default):  node import-fm-clients.js
 *   LIVE import:        CC_LIVE=1 CONFIRM=1 node import-fm-clients.js
 *
 * Steps (v3 API):
 *   1. GET  /contact_lists         → reuse list if the name already exists
 *   2. POST /contact_lists         → else create it, capture list_id
 *   3. POST /activities/contacts_json_import  (batched)  permission = implied
 *   4. report queued activity ids  (import runs async on CC's side)
 */
const fs = require('fs');
const path = require('path');
const cc = require('./cc-client.js');

const MAILABLE = path.join(
  process.env.HOME,
  'Projects/marketing-command-center/data/cc-import-fm-clients-mailable.json'
);
const LIST_NAME = 'DW FileMaker Clients (existing relationship)';
const BATCH = 500;
const LIVE = process.env.CC_LIVE === '1' && process.env.CONFIRM === '1';
const API = 'https://api.cc.email/v3';
// Testability seam (TK-11840): a fault can be INJECTED into the suppression guard only when the
// explicit test flag is set — a scheduled/LIVE job must NEVER pass CC_SUPPRESSION_TEST, so it can
// never silently measure a fixture. Used by the negative test that proves the guard reddens.
const SUPPRESSION_FILE = (process.env.CC_SUPPRESSION_TEST === '1' && process.env.CC_SUPPRESSION_FILE)
  ? process.env.CC_SUPPRESSION_FILE
  : path.join(__dirname, 'hygiene', 'suppression-list.json');
const SUPPRESSION_RESTORE = path.join(__dirname, 'hygiene', 'suppress-restore.json');

// Durable chronic-bounce guard (TK-11840): load the canonical suppression set so
// re-imports can NEVER re-add addresses that were removed for chronic bouncing
// (the TK-11387 regeneration bug — a full FM re-import re-added 5,253 bouncers).
// Reads the append-only registry first, falls back to the restore map's plan[].email.
function loadSuppressionSet() {
  const set = new Set();
  let source = null;
  let declared = null; // the file's OWN declared count — the "population" beside the observed set size
  try {
    const reg = JSON.parse(fs.readFileSync(SUPPRESSION_FILE, 'utf8'));
    for (const e of reg.emails || []) { const v = String(e).trim().toLowerCase(); if (v) set.add(v); }
    if (Number.isFinite(reg.count)) declared = reg.count;
    source = 'suppression-list.json';
  } catch {
    try {
      const rm = JSON.parse(fs.readFileSync(SUPPRESSION_RESTORE, 'utf8'));
      for (const r of rm.plan || []) { const v = String(r.email || '').trim().toLowerCase(); if (v) set.add(v); }
      if (Array.isArray(rm.plan)) declared = rm.plan.length;
      source = 'suppress-restore.json (fallback)';
    } catch { source = null; }
  }
  return { set, source, declared };
}

async function req(method, urlPath, body) {
  if (!LIVE) {
    console.log(`  [DRY-RUN] ${method} ${urlPath}` + (body ? ` (${JSON.stringify(body).length}b body)` : ''));
    return { __mock: true };
  }
  const token = await cc.getAccessToken();
  const res = await fetch(`${API}${urlPath}`, {
    method,
    headers: {
      Authorization: `Bearer ${token}`,
      Accept: 'application/json',
      ...(body ? { 'Content-Type': 'application/json' } : {}),
    },
    ...(body ? { body: JSON.stringify(body) } : {}),
  });
  const txt = await res.text();
  let j; try { j = txt ? JSON.parse(txt) : {}; } catch { j = { raw: txt }; }
  if (!res.ok) throw new Error(`${method} ${urlPath} → ${res.status}: ${txt}`);
  return j;
}

(async () => {
  const data = JSON.parse(fs.readFileSync(MAILABLE, 'utf8'));
  const rawContacts = data.import_data;

  // --- Durable chronic-bounce suppression guard (TK-11840) ---
  const { set: suppressed, source: supSource, declared: supDeclared } = loadSuppressionSet();
  if (suppressed.size === 0) {
    const msg = 'SUPPRESSION LIST EMPTY/UNREADABLE — refusing to import without the chronic-bounce guard.';
    if (LIVE) { console.error(`ABORT: ${msg} (expected ${SUPPRESSION_FILE})`); process.exit(1); }
    console.warn(`⚠️  ${msg} (dry-run continues; a LIVE run would ABORT)`);
  } else if (supDeclared != null && suppressed.size < Math.floor(supDeclared * 0.9)) {
    // Cody/TK-11840: a truncated-but-non-empty file is invisible to the size===0 check.
    // Carry the file's declared count beside the observed set size — a >10% shortfall means the
    // suppression set is DEGRADED (partial/corrupt write), so a LIVE import must fail closed too.
    const msg = `SUPPRESSION LIST DEGRADED — loaded ${suppressed.size} of a declared ${supDeclared} (`
      + `${((suppressed.size / supDeclared) * 100).toFixed(1)}%); refusing to import on a shrunken guard.`;
    if (LIVE) { console.error(`ABORT: ${msg} (expected ~${supDeclared} in ${SUPPRESSION_FILE})`); process.exit(1); }
    console.warn(`⚠️  ${msg} (dry-run continues; a LIVE run would ABORT)`);
  }
  const emailOf = (r) => String(r.email_address || r.email || '').trim().toLowerCase();
  const contacts = rawContacts.filter((r) => !suppressed.has(emailOf(r)));
  const excluded = rawContacts.length - contacts.length;

  console.log(`Mode: ${LIVE ? 'LIVE ✅' : 'DRY-RUN (set CC_LIVE=1 CONFIRM=1 to write)'}`);
  console.log(`Suppression source: ${supSource || 'NONE'} (${suppressed.size.toLocaleString()} addresses)`);
  console.log(`Mailable contacts: ${rawContacts.length.toLocaleString()} → ${contacts.length.toLocaleString()} after suppression (excluded ${excluded.toLocaleString()})`);

  // 1/2 — find or create the list
  let listId;
  if (LIVE) {
    const lr = await req('GET', '/contact_lists?limit=1000');
    const existing = (lr.lists || []).find((l) => l.name === LIST_NAME);
    if (existing) { listId = existing.list_id; console.log(`Reusing existing list ${listId}`); }
  }
  if (!listId) {
    const cr = await req('POST', '/contact_lists', {
      name: LIST_NAME,
      favorite: false,
      description: 'DW clients from FileMaker (existing relationship). Hygiene-passed + dead-domain filtered 2026-07-27.',
    });
    listId = LIVE ? cr.list_id : 'DRY-RUN-LIST-ID';
    console.log(`Created list ${listId}`);
  }

  // 3 — batched json import, permission basis = implied (existing customer)
  let queued = 0;
  // CC v3 contacts_json_import schema uses `email` (not `email_address`)
  const toImport = (r) => {
    const o = { email: r.email_address || r.email };
    if (r.company_name) o.company_name = r.company_name;
    if (r.phone) o.phone = r.phone;
    return o;
  };
  for (let i = 0; i < contacts.length; i += BATCH) {
    const chunk = contacts.slice(i, i + BATCH).map(toImport);
    const body = { import_data: chunk, list_ids: [listId] };
    const r = await req('POST', '/activities/contacts_json_import', body);
    queued += chunk.length;
    if (i % (BATCH * 10) === 0 || i + BATCH >= contacts.length) {
      console.log(`  queued ${queued.toLocaleString()}/${contacts.length.toLocaleString()}` +
        (r.activity_id ? ` (activity ${r.activity_id})` : ''));
    }
  }
  console.log(`\nDone. ${queued.toLocaleString()} contacts submitted to list ${listId}.`);
  console.log('CC processes imports asynchronously — verify final counts in the CC UI or via GET /activities.');
  if (!LIVE) console.log('\n(No data was written — this was a dry run.)');
})().catch((e) => { console.error('IMPORT FAILED:', e.message); process.exit(1); });