← back to Marketing Command Center

modules/ab-tests/index.js

396 lines

// A/B Tests module — plan + track email A/B campaign tests. The user defines
// two variants (subject + body/angle) and a holdout %; this module persists the
// test to data/ab-tests.json, simulates predicted opens/clicks per variant
// using a deterministic copy-quality heuristic, and surfaces a winner call.
//
// STAGED ONLY. No endpoint here triggers a real send. The "conclude" route
// just freezes the simulated/staged numbers + names a winner in the JSON.
// If real send numbers ever flow in (CTCT reporting webhook, GA4 event tap),
// the same record can be updated via PATCH /tests/:id/results without
// changing the front-end contract.
const fs = require('fs');
const path = require('path');
const crypto = require('crypto');

const brand = require('../../lib/brand.js');

const DATA_FILE = path.join(__dirname, '..', '..', 'data', 'ab-tests.json');

// ── persistence ─────────────────────────────────────────────────────────────
function readAll() {
  try { return JSON.parse(fs.readFileSync(DATA_FILE, 'utf8')); }
  catch { return []; }
}
function writeAll(rows) {
  fs.mkdirSync(path.dirname(DATA_FILE), { recursive: true });
  fs.writeFileSync(DATA_FILE, JSON.stringify(rows, null, 2));
}
function newId() {
  return 'abt_' + crypto.randomBytes(4).toString('hex');
}

// ── deterministic PRNG (mulberry32) so the same test re-simulates to the
// same numbers unless the user explicitly re-rolls with a new seed.
function mulberry32(seed) {
  let s = seed >>> 0;
  return function () {
    s |= 0; s = (s + 0x6d2b79f5) | 0;
    let t = Math.imul(s ^ (s >>> 15), 1 | s);
    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
  };
}
function hashSeed(s) {
  let h = 2166136261 >>> 0;
  for (let i = 0; i < s.length; i++) {
    h ^= s.charCodeAt(i);
    h = Math.imul(h, 16777619) >>> 0;
  }
  return h >>> 0;
}

// ── copy-quality heuristic ──────────────────────────────────────────────────
// Translates a variant's subject + body/angle into a baseline open-rate and
// click-rate multiplier. Designed to be transparent (each factor is named
// and exposed in the API response) so the user can see *why* a variant won.
const BRAND_KEYWORDS = brand.productLines.concat([
  'grasscloth', 'silk', 'cork', 'mural', 'metallic', 'memo', 'lookbook',
  'atelier', 'archive', 'limited', 'exclusive', 'designer', 'curated',
]);
const POWER_VERBS = ['discover', 'introducing', 'unveil', 'inside', 'meet', 'explore', 'request'];
const GIMMICKY = ['!!!', 'free', '$$$', 'act now', 'last chance', 'urgent', 'click here'];

function scoreCopy(variant) {
  const subj = String(variant.subject || '').trim();
  const body = String(variant.body || '').trim();
  const angle = String(variant.angle || '').trim().toLowerCase();
  const haystack = (subj + ' ' + body + ' ' + angle).toLowerCase();

  const factors = [];
  let openMult = 1.0;
  let clickMult = 1.0;

  // Subject length sweet spot 28-55 chars (editorial-feeling)
  const len = subj.length;
  if (len === 0) { openMult *= 0.4; factors.push({ name: 'empty-subject', open: -60, click: 0 }); }
  else if (len < 18) { openMult *= 0.88; factors.push({ name: 'subject-too-short', open: -12, click: 0 }); }
  else if (len > 70) { openMult *= 0.85; factors.push({ name: 'subject-too-long', open: -15, click: -3 }); clickMult *= 0.97; }
  else if (len >= 28 && len <= 55) { openMult *= 1.10; factors.push({ name: 'subject-length-ideal', open: +10, click: 0 }); }

  // Personalization-ish token (Designer, Studio, "your")
  if (/\b(designer|studio|atelier|your)\b/i.test(subj)) {
    openMult *= 1.06; factors.push({ name: 'audience-cue', open: +6, click: +1 }); clickMult *= 1.01;
  }

  // Question-mark headlines = curiosity bump
  if (/\?$/.test(subj)) { openMult *= 1.04; factors.push({ name: 'curiosity-question', open: +4, click: 0 }); }

  // Power verb in opening
  if (POWER_VERBS.some(v => new RegExp('^\\s*' + v + '\\b', 'i').test(subj))) {
    openMult *= 1.05; factors.push({ name: 'power-verb-open', open: +5, click: +1 }); clickMult *= 1.01;
  }

  // ALL CAPS penalty (>40% of subject in caps)
  const caps = (subj.match(/[A-Z]/g) || []).length;
  if (subj.length >= 10 && caps / subj.length > 0.4) {
    openMult *= 0.82; factors.push({ name: 'shouting-caps', open: -18, click: -5 }); clickMult *= 0.95;
  }

  // Gimmicky / spammy tokens (DW voice = "never gimmicky")
  let gimCount = 0;
  for (const g of GIMMICKY) if (haystack.includes(g)) gimCount++;
  if (gimCount > 0) {
    const pen = Math.min(0.9, 0.94 ** gimCount);
    openMult *= pen; clickMult *= pen;
    factors.push({ name: 'gimmicky-words', open: -((1 - pen) * 100).toFixed(0) * 1, click: -((1 - pen) * 100).toFixed(0) * 1, count: gimCount });
  }

  // Brand keyword in subject (grasscloth, silk, mural, etc.) — concrete product
  // tokens out-perform vague "wallpaper" subjects in DW historical data.
  let brandHit = false;
  for (const kw of BRAND_KEYWORDS) {
    if (subj.toLowerCase().includes(kw)) { brandHit = true; break; }
  }
  if (brandHit) { openMult *= 1.07; clickMult *= 1.04; factors.push({ name: 'concrete-product-token', open: +7, click: +4 }); }

  // Body length 80-700 chars = readable preview-text length
  if (body.length === 0) { clickMult *= 0.7; factors.push({ name: 'empty-body', open: 0, click: -30 }); }
  else if (body.length < 60) { clickMult *= 0.9; factors.push({ name: 'thin-body', open: 0, click: -10 }); }
  else if (body.length >= 80 && body.length <= 700) { clickMult *= 1.05; factors.push({ name: 'body-length-ideal', open: 0, click: +5 }); }

  // Body has a CTA-shaped phrase
  const ctaHit = brand.cta.some(c => haystack.includes(c.toLowerCase()))
    || /\b(request|explore|book|shop|view|see)\b/i.test(body);
  if (ctaHit) { clickMult *= 1.12; factors.push({ name: 'clear-cta', open: 0, click: +12 }); }

  // Angle tag bumps (editorial > promo > generic)
  if (angle === 'editorial' || angle === 'story') { openMult *= 1.05; clickMult *= 1.03; factors.push({ name: 'editorial-angle', open: +5, click: +3 }); }
  else if (angle === 'promo' || angle === 'sale') { openMult *= 0.95; clickMult *= 1.06; factors.push({ name: 'promo-angle', open: -5, click: +6 }); }
  else if (angle === 'new-arrival' || angle === 'launch') { openMult *= 1.04; clickMult *= 1.04; factors.push({ name: 'launch-angle', open: +4, click: +4 }); }

  return { openMult, clickMult, factors };
}

// ── simulator ───────────────────────────────────────────────────────────────
// Baseline DW historical-ish rates: 22% open, 3.2% click on a trade-leaning
// list. We split the audience between A / B (after holdout), apply each
// variant's quality multipliers, then add a small per-variant noise term
// derived from the test seed so re-rolls produce slightly different numbers.
const BASE_OPEN = 0.22;
const BASE_CLICK = 0.032;

function simulate(test) {
  const seed = hashSeed(test.id + ':' + (test.seedSalt || 'v1'));
  const rnd = mulberry32(seed);
  const audienceSize = Math.max(0, Number(test.audienceSize) || 0);
  const holdoutPct = Math.max(0, Math.min(50, Number(test.holdoutPct) || 0));

  const holdoutSize = Math.round(audienceSize * (holdoutPct / 100));
  const sendable = Math.max(0, audienceSize - holdoutSize);
  // Equal A/B split of the sendable pool. Round A down; B gets the remainder.
  const sentA = Math.floor(sendable / 2);
  const sentB = sendable - sentA;

  const scoreA = scoreCopy(test.variants[0]);
  const scoreB = scoreCopy(test.variants[1]);

  const noise = () => 0.94 + rnd() * 0.12; // ~±6%

  const variantResult = (sent, sc) => {
    const openRate = Math.max(0.01, Math.min(0.85, BASE_OPEN * sc.openMult * noise()));
    const clickRate = Math.max(0.001, Math.min(0.35, BASE_CLICK * sc.clickMult * sc.openMult * noise()));
    const opens = Math.round(sent * openRate);
    const clicks = Math.round(sent * clickRate);
    return {
      sent,
      opens, clicks,
      openRate: +openRate.toFixed(4),
      clickRate: +clickRate.toFixed(4),
      openMult: +sc.openMult.toFixed(3),
      clickMult: +sc.clickMult.toFixed(3),
      factors: sc.factors,
    };
  };

  // Holdout = no email at all — surface as zeros so the UI shows the bucket
  // size for "what we held back" comparison.
  const holdout = {
    sent: 0,
    held: holdoutSize,
    opens: 0,
    clicks: 0,
    openRate: 0,
    clickRate: 0,
  };

  const a = variantResult(sentA, scoreA);
  const b = variantResult(sentB, scoreB);

  return { variant_a: a, variant_b: b, holdout, simulatedAt: new Date().toISOString() };
}

// Two-proportion z-test (open-rate) for variant comparison. Returns the
// absolute lift, relative lift, and a coarse confidence bucket. We don't
// want to pretend this is a real significance result with mock data, so
// we expose the z-score honestly and tier it as low/med/high.
function compareWinner(results) {
  const a = results.variant_a, b = results.variant_b;
  if (!a.sent || !b.sent) return { winner: null, confidence: 'n/a', lift: 0, z: 0, metric: 'open-rate' };

  // Composite score: blended open × click, since either alone can mislead
  // (a clickbait subject might win opens but lose clicks).
  const compA = a.openRate * 0.6 + a.clickRate * 0.4;
  const compB = b.openRate * 0.6 + b.clickRate * 0.4;
  const winnerKey = compA === compB ? null : (compA > compB ? 'A' : 'B');

  // Stats on open-rate (more samples than clicks → tighter CI)
  const p1 = a.openRate, p2 = b.openRate;
  const n1 = a.sent, n2 = b.sent;
  const p = (a.opens + b.opens) / (n1 + n2);
  const se = Math.sqrt(p * (1 - p) * (1 / n1 + 1 / n2));
  const z = se > 0 ? Math.abs((p1 - p2) / se) : 0;

  // Map z to confidence: 1.28→80%, 1.65→90%, 1.96→95%, 2.58→99%
  let confidence = 'low';
  if (z >= 1.96) confidence = 'high';
  else if (z >= 1.28) confidence = 'med';

  const liftAbs = Math.abs(p1 - p2);
  const liftRel = winnerKey === 'A'
    ? (p2 > 0 ? (p1 - p2) / p2 : 0)
    : (p1 > 0 ? (p2 - p1) / p1 : 0);

  return {
    winner: winnerKey,
    confidence,
    metric: 'open-rate',
    liftAbsolute: +liftAbs.toFixed(4),
    liftRelative: +liftRel.toFixed(4),
    z: +z.toFixed(3),
    compositeA: +compA.toFixed(4),
    compositeB: +compB.toFixed(4),
  };
}

// ── validation ──────────────────────────────────────────────────────────────
function validateInput(body) {
  const errs = [];
  if (!body || typeof body !== 'object') return ['body must be an object'];
  if (!String(body.name || '').trim()) errs.push('name is required');
  if (!Array.isArray(body.variants) || body.variants.length !== 2) errs.push('variants must be an array of exactly 2');
  if (Array.isArray(body.variants)) {
    body.variants.forEach((v, i) => {
      const label = i === 0 ? 'A' : 'B';
      if (!v || typeof v !== 'object') errs.push(`variant ${label} must be an object`);
      else {
        if (!String(v.subject || '').trim()) errs.push(`variant ${label} subject is required`);
        if (!String(v.body || '').trim()) errs.push(`variant ${label} body is required`);
      }
    });
  }
  const aSize = Number(body.audienceSize);
  if (!Number.isFinite(aSize) || aSize < 100) errs.push('audienceSize must be ≥ 100');
  const hp = Number(body.holdoutPct);
  if (!Number.isFinite(hp) || hp < 0 || hp > 50) errs.push('holdoutPct must be 0–50');
  return errs;
}

function normalize(body) {
  return {
    name: String(body.name).trim(),
    description: String(body.description || '').trim(),
    audienceSegmentId: body.audienceSegmentId ? String(body.audienceSegmentId) : null,
    audienceSize: Math.round(Number(body.audienceSize)),
    holdoutPct: +Number(body.holdoutPct).toFixed(2),
    variants: [0, 1].map(i => ({
      key: i === 0 ? 'A' : 'B',
      subject: String(body.variants[i].subject).trim(),
      body: String(body.variants[i].body).trim(),
      angle: String(body.variants[i].angle || '').trim(),
    })),
  };
}

// ── module ──────────────────────────────────────────────────────────────────
module.exports = {
  id: 'ab-tests',
  title: 'A/B Tests',
  icon: '🔬',

  _scoreCopy: scoreCopy,
  _simulate: simulate,
  _compareWinner: compareWinner,

  mount(router) {
    const guard = (handler) => async (req, res) => {
      try { await handler(req, res); }
      catch (e) { res.status(500).json({ error: e.message }); }
    };

    // Constants — reference for the panel (angle options, baselines, gate copy)
    router.get('/meta', guard(async (_req, res) => {
      res.json({
        mock: true,
        staged: true,
        baseline: { openRate: BASE_OPEN, clickRate: BASE_CLICK, note: 'DW trade-leaning historical baseline' },
        angles: ['editorial', 'story', 'new-arrival', 'launch', 'promo', 'sale', 'memo-cta', 'lookbook'],
        holdoutRange: [0, 50],
        gateMessage: 'A/B Tests are STAGED — this module plans and tracks; it never triggers a live send. Approved variants must still ship through Constant Contact with Steve gating the send.',
      });
    }));

    // List all tests
    router.get('/tests', guard(async (_req, res) => {
      const rows = readAll();
      res.json({
        mock: true, staged: true,
        count: rows.length,
        tests: rows.map(t => ({
          id: t.id, name: t.name, status: t.status,
          createdAt: t.createdAt, concludedAt: t.concludedAt || null,
          audienceSize: t.audienceSize, holdoutPct: t.holdoutPct,
          winner: t.winner ? t.winner.winner : null,
          subjectA: t.variants?.[0]?.subject,
          subjectB: t.variants?.[1]?.subject,
        })),
      });
    }));

    // Single test detail
    router.get('/tests/:id', guard(async (req, res) => {
      const row = readAll().find(t => t.id === req.params.id);
      if (!row) return res.status(404).json({ error: 'test not found' });
      res.json({ mock: true, staged: true, test: row });
    }));

    // Create a new A/B test (staged — no send)
    router.post('/tests', guard(async (req, res) => {
      const errs = validateInput(req.body);
      if (errs.length) return res.status(400).json({ error: 'validation failed', issues: errs });

      const norm = normalize(req.body);
      const id = newId();
      const test = {
        id,
        ...norm,
        status: 'staged',          // staged → simulated → concluded
        createdAt: new Date().toISOString(),
        seedSalt: 'v1',
      };
      test.results = simulate(test);
      test.winner = compareWinner(test.results);

      const rows = readAll();
      rows.unshift(test);
      writeAll(rows);
      res.json({ mock: true, staged: true, test });
    }));

    // Re-simulate (re-roll noise term so the user can see variance)
    router.post('/tests/:id/simulate', guard(async (req, res) => {
      const rows = readAll();
      const idx = rows.findIndex(t => t.id === req.params.id);
      if (idx < 0) return res.status(404).json({ error: 'test not found' });
      const test = rows[idx];
      if (test.status === 'concluded') {
        return res.status(409).json({ error: 'test is concluded; results are frozen' });
      }
      // bump seed so simulation differs
      test.seedSalt = 'v' + (Date.now() % 100000);
      test.results = simulate(test);
      test.winner = compareWinner(test.results);
      rows[idx] = test;
      writeAll(rows);
      res.json({ mock: true, staged: true, test });
    }));

    // Conclude (freeze current numbers + winner). Still no send.
    router.post('/tests/:id/conclude', guard(async (req, res) => {
      const rows = readAll();
      const idx = rows.findIndex(t => t.id === req.params.id);
      if (idx < 0) return res.status(404).json({ error: 'test not found' });
      const test = rows[idx];
      if (test.status === 'concluded') {
        return res.json({ mock: true, staged: true, test, note: 'already concluded' });
      }
      if (!test.results) test.results = simulate(test);
      if (!test.winner) test.winner = compareWinner(test.results);
      test.status = 'concluded';
      test.concludedAt = new Date().toISOString();
      rows[idx] = test;
      writeAll(rows);
      res.json({ mock: true, staged: true, test });
    }));

    // Delete a test
    router.delete('/tests/:id', guard(async (req, res) => {
      const rows = readAll();
      const idx = rows.findIndex(t => t.id === req.params.id);
      if (idx < 0) return res.status(404).json({ error: 'test not found' });
      const [removed] = rows.splice(idx, 1);
      writeAll(rows);
      res.json({ mock: true, staged: true, deleted: removed.id });
    }));
  },
};