`server.js`

337 lines
#!/usr/bin/env node
// mfr-review-viewer — authenticated web viewer for SELECTING + REVIEWING broken
// DW catalog SKUs (the "DW# == Mfr SKU" corruption).
//
// READ-ONLY everywhere except ONE gated write: /api/stage drops a memo into
// ~/.claude/yolo-queue/pending-approval/. It NEVER writes to dw_unified,
// FileMaker, or Shopify. Staging is gated-only — it does not apply anything.

const express = require('express');
const { readFileSync, existsSync, writeFileSync, mkdirSync } = require('node:fs');
const { execFile, spawn } = require('node:child_process');
const path = require('node:path');
const os = require('node:os');

const PORT = process.env.PORT || 9786;
const ROOT = __dirname;
const QUEUE_FILE = path.join(ROOT, 'data', 'queue.jsonl');
const LOADER = path.join(ROOT, 'scripts', 'build-queue.mjs');
const ENRICHER = path.join(ROOT, 'scripts', 'enrich-suggestions.mjs');
const PENDING_DIR = path.join(os.homedir(), '.claude', 'yolo-queue', 'pending-approval');
const RESTORE_DIR = path.join(ROOT, 'data', 'restore');

// Basic Auth — admin / DW2024!, overridable via BASIC_AUTH="user:pass".
const [AUTH_USER, AUTH_PASS] = (process.env.BASIC_AUTH || 'admin:DW2024!').split(':');

const app = express();
app.use(express.json({ limit: '8mb' }));

// ---- Basic Auth gate (covers everything) --------------------------------
app.use((req, res, next) => {
  const hdr = req.headers.authorization || '';
  const [scheme, b64] = hdr.split(' ');
  if (scheme === 'Basic' && b64) {
    const [u, p] = Buffer.from(b64, 'base64').toString().split(':');
    if (u === AUTH_USER && p === AUTH_PASS) return next();
  }
  res.set('WWW-Authenticate', 'Basic realm="mfr-review-viewer"');
  return res.status(401).send('Authentication required');
});

// ---- Data ----------------------------------------------------------------
function loadQueue() {
  if (!existsSync(QUEUE_FILE)) return [];
  const lines = readFileSync(QUEUE_FILE, 'utf8').split('\n').filter((l) => l.trim());
  const rows = [];
  for (const l of lines) { try { rows.push(JSON.parse(l)); } catch { /* skip */ } }
  return rows;
}

// GET /api/queue — the full broken-SKU set + facet counts.
app.get('/api/queue', (_req, res) => {
  const rows = loadQueue();
  const facet = (key) => {
    const m = {};
    for (const r of rows) { const v = (r[key] === '' || r[key] == null) ? '(none)' : r[key]; m[v] = (m[v] || 0) + 1; }
    return m;
  };
  res.json({
    total: rows.length,
    rows,
    facets: {
      vendor: facet('vendor'),
      tier: facet('tier'),
      source: facet('source'),
      confidence: facet('confidence'),
    },
    tiers: facet('tier'),
  });
});

// GET /api/candidates?dw_sku=<sku> — lazy per-chip drill-down. READ-ONLY.
// Returns { filemaker:[...], unified:[...], fmError } for ONE dw_sku: every matching
// FileMaker (FmPro) master + every dw_unified row, so the reviewer can pick the right
// record and read its real mfr code. Never writes anywhere. Lazy ES-module import so a
// FileMaker/creds problem can't break the CJS server startup.
let _candidatesMod = null;
app.get('/api/candidates', async (req, res) => {
  const dwSku = String((req.query && req.query.dw_sku) || '').trim();
  if (!dwSku) return res.status(400).json({ filemaker: [], unified: [], fmError: 'dw_sku required' });
  try {
    if (!_candidatesMod) _candidatesMod = await import('./lib/candidates.mjs');
    const out = await _candidatesMod.candidatesForSku(dwSku);
    res.json(out);
  } catch (e) {
    // Degrade gracefully — never 500 out the whole viewer over a FM/creds hiccup.
    res.json({ filemaker: [], unified: [], fmError: e.message });
  }
});

// GET /api/fm-record-snapshot?recordId=<id> — READ-ONLY full-field dump of ONE
// WALLPAPER master, used to snapshot a record before proposing its deletion. Never
// writes to FileMaker. Lazy ES-module import (a FM/creds hiccup can't break startup).
let _snapMod = null;
app.get('/api/fm-record-snapshot', async (req, res) => {
  const recordId = String((req.query && req.query.recordId) || '').trim();
  if (!recordId) return res.status(400).json({ ok: false, error: 'recordId required' });
  try {
    if (!_snapMod) _snapMod = await import('./lib/fm-record-snapshot.mjs');
    const snap = await _snapMod.snapshotRecord(recordId);
    res.json({ ok: true, ...snap });
  } catch (e) {
    res.json({ ok: false, error: e.message, recordId });
  }
});

// POST /api/rebuild — re-run the loader (behind auth). READ-ONLY vs dw_unified.
app.post('/api/rebuild', (_req, res) => {
  execFile('node', [LOADER], { maxBuffer: 1024 * 1024 * 256 }, (err, stdout, stderr) => {
    if (err) return res.status(500).json({ ok: false, error: stderr || err.message });
    const rows = loadQueue();
    // Auto-refill the FileMaker-derived real-mfr suggestions after a rebuild so the list
    // never goes back to blank. Detached + unref'd so it outlives this request; resumable.
    let enriching = false;
    try { const c = spawn('node', [ENRICHER], { cwd: ROOT, detached: true, stdio: 'ignore' }); c.unref(); enriching = true; } catch { /* best-effort */ }
    res.json({ ok: true, total: rows.length, enriching, log: (stdout || '').trim().split('\n').slice(-6) });
  });
});

// GET /api/vendor-plan?vendor=<name> — run the READ-ONLY vendor analyzer and return
// the aggregated plan JSON + counts. Touches NOTHING (psql SELECTs + FM _find reads).
// Powers the viewer's "⚙ Sweep this vendor" modal. Lazy ES-module import.
const ANALYZER = path.join(ROOT, 'scripts', 'vendor-sweep-analyze.mjs');
app.get('/api/vendor-plan', (req, res) => {
  const vendor = String((req.query && req.query.vendor) || '').trim();
  if (!vendor) return res.status(400).json({ ok: false, error: 'vendor required' });
  const args = [ANALYZER, vendor];
  if (req.query.limit) args.push('--limit', String(parseInt(req.query.limit, 10) || 25));
  execFile('node', args, { maxBuffer: 1024 * 1024 * 64, timeout: 15 * 60 * 1000 }, (err, stdout, stderr) => {
    if (err) return res.status(500).json({ ok: false, error: stderr || err.message });
    // The analyzer wrote data/plans/<slug>-<ISO>.json; find the newest for this vendor.
    try {
      const plansDir = path.join(ROOT, 'data', 'plans');
      const slug = vendor.replace(/[^A-Za-z0-9]+/g, '-').replace(/(^-|-$)/g, '');
      const files = existsSync(plansDir)
        ? require('node:fs').readdirSync(plansDir).filter((f) => f.startsWith(slug + '-') && f.endsWith('.json'))
        : [];
      files.sort();
      const newest = files[files.length - 1];
      if (!newest) return res.json({ ok: true, vendor, counts: {}, plan: [], planFile: null, log: (stdout || '').trim().split('\n').slice(-3) });
      const planPath = path.join(plansDir, newest);
      const data = JSON.parse(readFileSync(planPath, 'utf8'));
      res.json({ ok: true, vendor, counts: data.counts || {}, plan: data.plan || [],
        planFile: path.join('data', 'plans', newest), log: (stdout || '').trim().split('\n').slice(-3) });
    } catch (e) { res.status(500).json({ ok: false, error: e.message }); }
  });
});

// POST /api/vendor-stage — write the gated pending-approval memo for a vendor sweep
// (SAFE DEFAULT). Reads the plan file the analyzer produced + snapshots each delete first,
// then reuses the SAME /api/stage memo machinery via the deletions[] path. NO delete.
app.post('/api/vendor-stage', async (req, res) => {
  const planFile = String((req.body && req.body.planFile) || '').trim();
  if (!planFile) return res.status(400).json({ ok: false, error: 'planFile required' });
  const abs = path.isAbsolute(planFile) ? planFile : path.join(ROOT, planFile);
  if (!existsSync(abs)) return res.status(400).json({ ok: false, error: 'plan file not found' });
  let plan;
  try { const d = JSON.parse(readFileSync(abs, 'utf8')); plan = d.plan || d; }
  catch (e) { return res.status(400).json({ ok: false, error: 'bad plan JSON: ' + e.message }); }
  // Map the analyzer plan onto the deletions[] shape /api/stage already handles.
  req.body.deletions = (plan || []).map((e) => ({
    dw_sku: e.dw_sku, keepRecordId: e.keepRecordId, keepMfr: e.realMfr,
    keepNote: 'vendor-sweep (mfr-master-sweep)', deleteRecordIds: e.deleteRecordIds || [],
  }));
  req.body.selected = [];
  return stageHandler(req, res); // reuse the exact same gated-memo + snapshot code
});

// POST /api/vendor-execute — the GATED DESTRUCTIVE "Execute now" path. Requires a typed
// confirm token `APPROVE <vendor>` in the body. Runs the executor with --apply against the
// plan file. This is a real canonical FileMaker delete + mfr-write; it is only reachable
// after the user types the exact APPROVE token in the modal.
const EXECUTOR = path.join(ROOT, 'scripts', 'execute-sweep.mjs');
app.post('/api/vendor-execute', (req, res) => {
  const vendor = String((req.body && req.body.vendor) || '').trim();
  const planFile = String((req.body && req.body.planFile) || '').trim();
  const confirm = String((req.body && req.body.confirm) || '').trim();
  const max = parseInt((req.body && req.body.max), 10) || 500;
  if (!vendor || !planFile) return res.status(400).json({ ok: false, error: 'vendor + planFile required' });
  if (confirm !== `APPROVE ${vendor}`) {
    return res.status(403).json({ ok: false, error: `confirmation mismatch — type exactly: APPROVE ${vendor}` });
  }
  const abs = path.isAbsolute(planFile) ? planFile : path.join(ROOT, planFile);
  if (!existsSync(abs)) return res.status(400).json({ ok: false, error: 'plan file not found' });
  const args = [EXECUTOR, '--plan', abs, '--apply', '--max', String(max), '--ticket', 'TK-10909', '--agent', 'vp-dw-commerce'];
  execFile('node', args, { maxBuffer: 1024 * 1024 * 64, timeout: 30 * 60 * 1000, env: process.env },
    (err, stdout, stderr) => {
      if (err) return res.status(500).json({ ok: false, error: stderr || err.message, log: (stdout || '').split('\n').slice(-20) });
      res.json({ ok: true, vendor, applied: true, log: (stdout || '').trim().split('\n').slice(-30) });
    });
});

// POST /api/stage — THE ONLY WRITE. Gated memo (+ reversible restore snapshots),
// never a canonical/live delete. Accepts:
//   selected:  [ { dw_sku, bad_mfr, confirmed_real_mfr, vendor, source } ]  (mfr-repair rows)
//   deletions: [ { dw_sku, keepRecordId, keepMfr, keepNote, deleteRecordIds:[...] } ]
//              (per-SKU duplicate-master pruning proposals)
//
// For EACH delete recordId the server FIRST writes a full-field snapshot to
// data/restore/<recordId>-<ISO>.json (READ-ONLY FM read) so an approved delete is
// REVERSIBLE (recreatable). The viewer performs ZERO FileMaker deletes and ZERO
// canonical writes — it only reads FM + writes the restore JSON + the memo. The actual
// delete is a SEPARATE Steve-approved step (there is no auto-executor here).
async function snapshotForRestore(recordId, iso) {
  try {
    if (!_snapMod) _snapMod = await import('./lib/fm-record-snapshot.mjs');
    const snap = await _snapMod.snapshotRecord(recordId);
    if (!existsSync(RESTORE_DIR)) mkdirSync(RESTORE_DIR, { recursive: true });
    const file = `${recordId}-${iso}.json`;
    writeFileSync(path.join(RESTORE_DIR, file), JSON.stringify(snap, null, 2));
    return { recordId, restorePath: path.join('data', 'restore', file), ok: true,
      fieldCount: Object.keys(snap.fieldData || {}).length };
  } catch (e) {
    // A snapshot must NOT be silently skipped — a delete without a saved snapshot is
    // NOT reversible, so we record the failure and refuse to list it as restorable.
    return { recordId, restorePath: null, ok: false, error: e.message };
  }
}

async function stageHandler(req, res) {
  const selected = Array.isArray(req.body && req.body.selected) ? req.body.selected : [];
  const deletions = Array.isArray(req.body && req.body.deletions) ? req.body.deletions : [];
  if (!selected.length && !deletions.length) {
    return res.status(400).json({ ok: false, error: 'nothing selected (no rows and no deletions)' });
  }

  const iso = new Date().toISOString().replace(/[:.]/g, '-');
  const memoName = `mfr-repair-${iso}.md`;
  const memoPath = path.join(PENDING_DIR, memoName);

  const esc = (s) => String(s == null ? '' : s).replace(/\|/g, '\\|').replace(/[\r\n]+/g, ' ');
  const inline = (s) => String(s == null ? '' : s).replace(/[\r\n]+/g, ' ').trim();

  // ---- snapshot every delete-candidate FIRST (reversibility record) ----------
  // Map: recordId -> { restorePath, ok, error } so the memo can cite each snapshot.
  const snapMap = {};
  let totalDeletes = 0, snapOk = 0, snapFail = 0;
  for (const d of deletions) {
    const ids = Array.isArray(d.deleteRecordIds) ? d.deleteRecordIds.map(String) : [];
    for (const rid of ids) {
      totalDeletes++;
      if (snapMap[rid]) continue;                 // dedupe across SKUs (shouldn't overlap, but safe)
      const s = await snapshotForRestore(rid, iso);
      snapMap[rid] = s;
      s.ok ? snapOk++ : snapFail++;
    }
  }
  const skuWithDeletes = deletions.filter((d) => Array.isArray(d.deleteRecordIds) && d.deleteRecordIds.length).length;

  // ---- build the memo --------------------------------------------------------
  const parts = [];
  parts.push('# MFR SKU Repair — staged for approval');
  parts.push('');
  parts.push(`**Staged:** ${new Date().toISOString()}`);
  if (selected.length) parts.push(`**Mfr-repair rows:** ${selected.length}`);
  if (skuWithDeletes) {
    parts.push(`**Proposes ${totalDeletes} master deletion(s) across ${skuWithDeletes} SKU(s) — REVERSIBLE via saved field snapshots; APPROVE to execute.**`);
    if (snapFail) parts.push(`> ⚠ ${snapFail} of ${totalDeletes} delete snapshot(s) FAILED to capture — those deletes are NOT reversible and must NOT be executed (see per-SKU notes).`);
  }
  parts.push('');
  parts.push('**Staged by mfr-review-viewer — NOT deleted. Nothing was written to dw_unified, FileMaker, or Shopify by the viewer.** The actual delete is a SEPARATE Steve-approved step.');
  parts.push('');

  // -- mfr-repair table (unchanged behavior) --
  if (selected.length) {
    parts.push('## Mfr-code repairs');
    parts.push('');
    parts.push(`${selected.length} broken "DW# == Mfr SKU" catalog row(s) selected for mfr-code repair. Each row's "Confirmed Real Mfr" is the reviewer-edited value from mfr-review-viewer.`);
    parts.push('');
    parts.push('| DW SKU | Current Bad Mfr | Confirmed Real Mfr | Vendor | Source |');
    parts.push('|---|---|---|---|---|');
    for (const r of selected) {
      parts.push(`| ${esc(r.dw_sku)} | ${esc(r.bad_mfr)} | ${esc(r.confirmed_real_mfr)} | ${esc(r.vendor)} | ${esc(r.source)} |`);
    }
    parts.push('');
    parts.push('**APPROVE to apply mfr-code repairs to dw_unified/FileMaker/Shopify — reversible via saved old→new map** (old mfr = "Current Bad Mfr"; new mfr = "Confirmed Real Mfr").');
    parts.push('');
  }

  // -- duplicate-master pruning proposals (KEEP / DELETE per SKU) --
  if (skuWithDeletes) {
    parts.push('## Duplicate-master pruning');
    parts.push('');
    parts.push('Per SKU: KEEP the one correct master, DELETE the duplicate masters. Each DELETE has a saved full-field restore snapshot so an approved delete is reversible (recreatable from the snapshot).');
    parts.push('');
    for (const d of deletions) {
      const ids = Array.isArray(d.deleteRecordIds) ? d.deleteRecordIds.map(String) : [];
      if (!ids.length) continue;
      parts.push(`### ${esc(d.dw_sku)}`);
      const keepBits = [];
      if (d.keepRecordId) keepBits.push(`master ${inline(d.keepRecordId)}`);
      if (d.keepMfr) keepBits.push(`confirmed mfr: ${inline(d.keepMfr)}`);
      if (d.keepNote) keepBits.push(`note: ${inline(d.keepNote)}`);
      parts.push(`KEEP  ${keepBits.length ? keepBits.join('  ') : '(none chosen — reviewer did not mark a keep)'}`);
      const delLine = ids.map((rid) => inline(rid)).join(', ');
      parts.push(`DELETE masters: ${delLine}  (duplicates)`);
      parts.push('  restore snapshots:');
      for (const rid of ids) {
        const s = snapMap[rid] || {};
        if (s.ok) parts.push(`  - ${inline(rid)} → ${s.restorePath}  (${s.fieldCount} fields)`);
        else parts.push(`  - ${inline(rid)} → ⚠ SNAPSHOT FAILED (${inline(s.error) || 'unknown'}) — NOT reversible, do NOT delete`);
      }
      parts.push('');
    }
    parts.push('**APPROVE to execute the DELETEs against FileMaker (WALLPAPER) — each is REVERSIBLE by re-creating the record from its saved data/restore/<recordId>-<ISO>.json snapshot.**');
    parts.push('');
  }

  const memo = parts.join('\n');

  try {
    if (!existsSync(PENDING_DIR)) mkdirSync(PENDING_DIR, { recursive: true });
    writeFileSync(memoPath, memo);
  } catch (e) {
    return res.status(500).json({ ok: false, error: e.message });
  }
  res.json({
    ok: true,
    staged: selected.length,
    deletions: totalDeletes,
    skuWithDeletes,
    snapshotsOk: snapOk,
    snapshotsFailed: snapFail,
    memo: memoPath,
    memoName,
  });
}
app.post('/api/stage', stageHandler);

// Silence the cosmetic /favicon.ico 404 (no-content icon).
app.get('/favicon.ico', (_req, res) => res.status(204).end());

app.use(express.static(path.join(ROOT, 'public')));

app.listen(PORT, '127.0.0.1', () => {
  console.log(`mfr-review-viewer listening on http://127.0.0.1:${PORT}`);
});