`scripts/batch-regen-settlement.js`

211 lines
#!/usr/bin/env node
/**
 * batch-regen-settlement.js — background batch driver for the 284 settlement-
 * blocked designs (2026-06-01, Steve-approved).
 *
 * Walks /tmp/regen_worklist.json. For each design:
 *   1. derives a concept question from the prompt subject (category-aware)
 *   2. runs scripts/regen-4gate-settlement.sh (seam + flat-color + concept-vision
 *      + SETTLEMENT-OK), capped at MAX_ROLLS rolls
 *   3. on a 4-gate winner → promotes it to STAGED (is_published=FALSE) by POSTing
 *      /api/seam-debug/heal/approve {src_id, heal_id}. Because every original is
 *      currently unpublished, the heal inherits is_published=FALSE — NOTHING goes
 *      live. The original becomes user_removed=TRUE (recoverable).
 *   4. on cap-reached → logs ABANDONED (e.g. legally-blocked Merian damask).
 *
 * GUARDRAILS (Steve's hard rules — enforced here):
 *   - NEVER publishes live. Winners are staged only; the heal/approve route
 *     inherits the original's (unpublished) state.
 *   - NEVER overrides the settlement gate. The 4th gate is a genuine OK.
 *   - Per-design MAX_ROLLS cap so an unwinnable design is abandoned, not an
 *     infinite Replicate burn.
 *   - Cumulative roll count + est cost (~$0.006/roll) tracked in the ledger.
 *
 * Ledger: data/batch-regen-settlement-ledger.jsonl (one JSON line per design).
 * Live log: /tmp/batch-regen-settlement.live.log (driver-level progress).
 *
 * Resumable: skips any design already terminal (winner|abandoned) in the ledger.
 *
 * Usage:
 *   node scripts/batch-regen-settlement.js               # full worklist
 *   node scripts/batch-regen-settlement.js --limit 3     # smoke test first N
 *   node scripts/batch-regen-settlement.js --dry         # plan only, no gen
 *
 * Concurrency (DTD verdict C, 2026-06-01): runs CONCURRENCY designs in parallel
 * (default 2). The id-grab race is already eliminated by reading the new id from
 * generate_designs.js's deterministic stdout 'IDs:' line (NOT MAX(id)), so the
 * gen lock is unnecessary for correctness — the worker no longer takes it. The
 * 2-design cap is a pure concurrency budget: it cuts wall-clock ~2x vs serial
 * while leaving Replicate/CPU headroom for the standing curator generator to
 * interleave on its 30-min tick.
 */
'use strict';
const fs = require('fs');
const path = require('path');
const { spawn, execSync } = require('child_process');

const ROOT = path.join(__dirname, '..');
const WORKLIST = process.env.WORKLIST || '/tmp/regen_worklist.json';
const LEDGER = path.join(ROOT, 'data', 'batch-regen-settlement-ledger.jsonl');
const LIVE_LOG = '/tmp/batch-regen-settlement.live.log';
const WORKER = path.join(ROOT, 'scripts', 'regen-4gate-settlement.sh');
const PORT = process.env.PORT || '9905';
const MAX_ROLLS = parseInt(process.env.MAX_ROLLS || '20', 10);
const COST_PER_ROLL = parseFloat(process.env.COST_PER_ROLL || '0.006'); // Replicate SDXL est.
const CONCURRENCY = parseInt(process.env.CONCURRENCY || '2', 10); // DTD-C: 2 designs in flight

const args = process.argv.slice(2);
const LIMIT = args.includes('--limit') ? parseInt(args[args.indexOf('--limit') + 1], 10) : Infinity;
const DRY = args.includes('--dry');

function log(msg) {
  const line = `[${new Date().toISOString()}] ${msg}`;
  console.log(line);
  try { fs.appendFileSync(LIVE_LOG, line + '\n'); } catch (_) {}
}
function ledgerAppend(obj) {
  fs.appendFileSync(LEDGER, JSON.stringify(obj) + '\n');
}
function alreadyDone() {
  const done = new Set();
  if (!fs.existsSync(LEDGER)) return done;
  for (const ln of fs.readFileSync(LEDGER, 'utf8').split('\n')) {
    if (!ln.trim()) continue;
    try { const o = JSON.parse(ln); if (o.outcome === 'winner' || o.outcome === 'abandoned') done.add(String(o.src_id)); } catch (_) {}
  }
  return done;
}

// Derive a concept question from the prompt + category. Drunk/zoo categories →
// "is there a recognizable animal". Damask/scenic/floral → "does the motif match
// the intended subject" (the pattern style itself). The concept gate just needs
// the rendered tile to actually contain the prompted subject, not a blob.
function conceptQuestion(category, prompt) {
  const cat = (category || '').toLowerCase();
  const p = (prompt || '').toLowerCase();
  if (cat.includes('drunk') || cat.includes('zoo') || cat.includes('animal')) {
    return 'Does this wallpaper tile contain at least one clearly recognizable animal (a mammal, bird, reptile, or zoo-style creature with a discernible body/head/limbs) as a real subject — NOT just an abstract blob, smear, or plants/leaves alone?';
  }
  if (cat.includes('damask')) {
    return 'Does this wallpaper tile show a clear, deliberate repeating damask/ornamental motif (a structured symmetrical decorative pattern) — NOT an abstract blob, noise, or empty dead space?';
  }
  if (cat.includes('floral') || p.includes('floral') || p.includes('flower') || p.includes('tulip') || p.includes('rose')) {
    return 'Does this wallpaper tile contain clearly recognizable flowers/botanical motifs arranged as a deliberate repeating pattern — NOT an abstract blob or empty dead space?';
  }
  if (cat.includes('scenic') || cat.includes('mural')) {
    return 'Does this tile contain a coherent, deliberately composed decorative scene/motif matching a wallpaper design — NOT an abstract blob, smear, or empty dead space?';
  }
  // generic fallback — just demand a real deliberate motif, not a blob
  return 'Does this wallpaper tile contain a clear, deliberate repeating decorative motif — NOT an abstract blob, smear, noise, or empty dead space?';
}

function promoteStaged(srcId, healId) {
  // localhost → admin-gate grants full admin with no token. Stages the winner
  // (heal inherits the original's UNPUBLISHED state → never goes live).
  const body = JSON.stringify({ src_id: Number(srcId), heal_id: Number(healId) });
  const out = execSync(
    `curl -s -X POST "http://127.0.0.1:${PORT}/api/seam-debug/heal/approve" ` +
    `-H 'Content-Type: application/json' -d '${body}'`,
    { encoding: 'utf8', timeout: 30000 }
  );
  let j; try { j = JSON.parse(out); } catch (_) { throw new Error('approve route returned non-JSON: ' + out.slice(0, 200)); }
  if (!j.ok) throw new Error('approve failed: ' + out.slice(0, 200));
  if (j.promoted_to_published) throw new Error('GUARDRAIL TRIPPED: winner was promoted to PUBLISHED — original was published. Aborting.');
  return j;
}

// Run ONE design through the worker (async). Resolves with its ledger record.
function runDesign(item, idx) {
  return new Promise((resolve) => {
    const srcId = String(item.id);
    const cq = conceptQuestion(item.category, item.prompt);
    log(`[${idx}] START #${srcId} cat=${item.category} verdict=${item.verdict} — regen (cap ${MAX_ROLLS})`);
    const T0 = Date.now();
    const child = spawn('bash', [WORKER, srcId], {
      cwd: ROOT,
      env: { ...process.env, SRC_ID: srcId, CATEGORY: item.category, MAX_ROLLS: String(MAX_ROLLS), CONCEPT_Q: cq, GEN_BACKEND: process.env.GEN_BACKEND || 'replicate' },
      stdio: ['ignore', 'inherit', 'inherit'],
    });
    const killT = setTimeout(() => { try { child.kill('SIGTERM'); } catch (_) {} }, 1000 * 60 * 120); // 120 min ceiling/design
    child.on('exit', (code, signal) => {
      clearTimeout(killT);
      const dtMin = ((Date.now() - T0) / 60000).toFixed(1);
      let rolls = 0;
      try { rolls = parseInt(fs.readFileSync(`/tmp/regen-4gate-${srcId}.rolls`, 'utf8').trim(), 10) || 0; } catch (_) {}
      let outcome, healId = null, reason = '';
      const winnerFile = `/tmp/regen-4gate-${srcId}.winner`;
      if (code === 0 && fs.existsSync(winnerFile)) {
        healId = parseInt(fs.readFileSync(winnerFile, 'utf8').trim(), 10);
        try {
          promoteStaged(srcId, healId);
          outcome = 'winner';
          reason = `staged heal #${healId} (is_published=FALSE), original user_removed`;
        } catch (e) { outcome = 'error'; reason = 'promote failed: ' + e.message; }
      } else if (code === 2) {
        outcome = 'abandoned';
        reason = `hit ${MAX_ROLLS}-roll cap without settlement-OK winner (likely legally/structurally unwinnable)`;
      } else {
        outcome = 'error';
        reason = `worker exit ${code}${signal ? ' signal=' + signal : ''}`;
      }
      resolve({ srcId, item, outcome, healId, rolls, dtMin: Number(dtMin), reason });
    });
  });
}

async function main() {
  if (!fs.existsSync(WORKLIST)) { console.error('worklist not found: ' + WORKLIST); process.exit(1); }
  const work = JSON.parse(fs.readFileSync(WORKLIST, 'utf8'));
  const done = alreadyDone();

  // build the queue (skip non-integer ids + already-terminal designs + --limit)
  const queue = [];
  for (const item of work) {
    const srcId = String(item.id);
    if (!/^[0-9]+$/.test(srcId)) { log(`SKIP non-integer id "${srcId}"`); continue; }
    if (done.has(srcId)) continue;
    queue.push(item);
    if (queue.length >= LIMIT) break;
  }

  let totalRolls = 0, winners = 0, abandoned = 0, errors = 0, processed = 0;
  log(`=== batch-regen-settlement START · ${work.length} in worklist · ${queue.length} to process · ${done.size} already done · CONCURRENCY=${CONCURRENCY} · MAX_ROLLS=${MAX_ROLLS} · cost/roll≈$${COST_PER_ROLL} ===`);

  if (DRY) {
    queue.forEach((item, i) => log(`  DRY [${i + 1}] #${item.id} cat=${item.category} concept="${conceptQuestion(item.category, item.prompt).slice(0, 60)}…"`));
    log(`=== DRY DONE · ${queue.length} planned ===`);
    return;
  }

  let cursor = 0;
  function finish(rec) {
    processed++;
    totalRolls += rec.rolls;
    if (rec.outcome === 'winner') winners++;
    else if (rec.outcome === 'abandoned') abandoned++;
    else errors++;
    const estCost = +(totalRolls * COST_PER_ROLL).toFixed(3);
    ledgerAppend({
      ts: new Date().toISOString(), src_id: Number(rec.srcId), category: rec.item.category,
      original_verdict: rec.item.verdict, outcome: rec.outcome, heal_id: rec.healId,
      rolls: rec.rolls, dt_min: rec.dtMin, cumulative_rolls: totalRolls, est_cost_usd: estCost, reason: rec.reason,
    });
    log(`  → ${rec.outcome.toUpperCase()} #${rec.srcId}${rec.healId ? ' heal=' + rec.healId : ''} · rolls=${rec.rolls} · ${rec.dtMin}min · [${processed}/${queue.length}] cum_rolls=${totalRolls} · est_cost≈$${estCost} · ${rec.reason}`);
  }

  // simple N-slot pool
  async function worker() {
    while (cursor < queue.length) {
      const myIdx = cursor++;
      const rec = await runDesign(queue[myIdx], myIdx + 1);
      finish(rec);
    }
  }
  await Promise.all(Array.from({ length: Math.min(CONCURRENCY, queue.length) }, () => worker()));

  const estCost = +(totalRolls * COST_PER_ROLL).toFixed(3);
  log(`=== DONE · processed=${processed} winners=${winners} abandoned=${abandoned} errors=${errors} · total_rolls=${totalRolls} · est_cost≈$${estCost} ===`);
}

main();