`scripts/extract-arena-ideas.cjs`

98 lines
#!/usr/bin/env node
// extract-arena-ideas.cjs — pull every cartoon idea out of a finished Model Arena battle (TK-12241). $0, local.
// Usage: node scripts/extract-arena-ideas.cjs [battleDir]   (default ~/Projects/model-arena/data/artifacts/7b062a7d42e5)
// Each artifact is a model-written HTML page; some build their cards from an inline <script> array
// (const ideas/cards = [...]), others hard-code .card DOM. Every page is rendered in headless Chromium
// and read BOTH ways: the JS array when present (authoritative), else the rendered .card DOM.
// Output: data/arena-ideas.json — deduped, with dropped ideas listed + reasons. Existing curation
// status (approved/deleted) is carried over by id on re-extract.
const path = require('path');
const fs = require('fs');
const crypto = require('crypto');
const { chromium } = require(process.env.PLAYWRIGHT_PATH || path.join(process.env.HOME, 'Projects/jevrun-runner/node_modules/playwright'));

const DIR = process.argv[2] || path.join(process.env.HOME, 'Projects/model-arena/data/artifacts/7b062a7d42e5');
const BATTLE = path.basename(DIR);
const OUT = path.join(__dirname, '..', 'data', 'arena-ideas.json');
const NAME = new RegExp(['con', 'rad'].join(''), 'i');
// Drop rules: real people, brands / franchises, named artists, style-mimicry, and caricature risk.
const DROP = [
  [/in the style of|style of [A-Z]/i, 'asks for an artist style'],
  [NAME, 'names the research subject'],
  [/\b(trump|biden|obama|clinton|bush|nixon|reagan|kennedy|carter|putin|musk|bezos|zuckerberg|pelosi|mcconnell|harris|desantis|newsom|schwarzenegger)\b/i, 'names a real person'],
  [/\b(uber|lyft|instagram|twitter|tiktok|facebook|meta|google|amazon|apple|tesla|netflix|disney|starbucks|mcdonald'?s?|coca-?cola|pepsi|walmart|star wars|middle[- ]earth|tolkien|marvel|pixar|lego|iphone|youtube)\b/i, 'names a brand / franchise'],
  [/\b(picasso|warhol|dali|van gogh|banksy|rockwell|seuss|schulz)\b/i, 'names an artist'],
  [/\b(black|white|asian|latino|jewish|muslim) (man|woman|person|people)'?s? face\b|face on the pendulum/i, 'racial caricature risk'],
];

const clean = (s) => String(s == null ? '' : s).replace(/\s+/g, ' ').trim();
const unquote = (s) => clean(s).replace(/^["“'‘]+|["”'’]+$/g, '');
const modelOf = (f) => path.basename(f, '.html').replace(/^(mac2|mlx)__/, '');

(async () => {
  const files = fs.readdirSync(DIR).filter(f => f.endsWith('.html')).sort();
  const browser = await chromium.launch();
  const raw = [];
  const perModel = {};
  for (const f of files) {
    const page = await browser.newPage();
    await page.goto('file://' + path.join(DIR, f), { waitUntil: 'load' });
    const got = await page.evaluate(() => {
      // eslint-disable-next-line no-undef
      const arr = (typeof ideas !== 'undefined' && Array.isArray(ideas)) ? ideas : (typeof cards !== 'undefined' && Array.isArray(cards) && cards.length && !(cards[0] instanceof Element)) ? cards : null;
      if (arr) return { via: 'script-array', rows: arr.map(o => ({ theme: o.theme, title: o.title, caption: o.caption, scene: o.scene, why: o.why || o.whyFunny || o.reason, drawability: o.drawability })) };
      const q = (el, sel) => { const n = el.querySelector(sel); return n ? n.textContent : ''; };
      return { via: 'dom', rows: [...document.querySelectorAll('.card')].map(c => ({
        theme: q(c, '.theme-chip, .theme'), title: q(c, '.card-title, h2, h3'), caption: q(c, '.caption'), scene: q(c, '.scene'),
        why: q(c, '.funny-reason, .whys, .why'), drawability: c.getAttribute('data-drawability') || q(c, '.drawability, .score'),
      })) };
    });
    const mtime = fs.statSync(path.join(DIR, f)).mtime.toISOString();
    perModel[modelOf(f)] = { file: f, via: got.via, extracted: got.rows.length };
    for (const r of got.rows) raw.push({ ...r, model: modelOf(f), created_at: mtime });
    await page.close();
  }
  await browser.close();

  const prev = fs.existsSync(OUT) ? JSON.parse(fs.readFileSync(OUT, 'utf8')) : { items: [] };
  const prevById = Object.fromEntries(prev.items.map(i => [i.id, i]));
  const seen = new Map();
  const items = [];
  const dropped = [];
  for (const r of raw) {
    let caption = unquote(r.caption);
    const notes = [];
    if (/[　-鿿]/.test(caption + r.scene)) { caption = caption.replace(/授权/g, 'authorized ').replace(/[　-鿿]+/g, '').replace(/\s+/g, ' ').trim(); notes.push('garbled CJK token cleaned'); }
    const it = {
      model: r.model, theme: clean(r.theme).replace(/&amp;/g, '&'), title: clean(r.title), caption,
      scene: clean(r.scene).replace(/[　-鿿]+/g, ''), why: clean(r.why), drawability: Number(clean(r.drawability)) || null,
      created_at: r.created_at,
    };
    const blob = [it.title, it.caption, it.scene, it.why].join(' ');
    const hit = DROP.find(([re]) => re.test(blob));
    if (!it.title || !it.scene) { dropped.push({ model: it.model, title: it.title, reason: 'incomplete (no title/scene)' }); continue; }
    if (hit) { dropped.push({ model: it.model, title: it.title, reason: hit[1] }); continue; }
    const key = it.title.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
    const ckey = it.caption.toLowerCase().replace(/[^a-z0-9]+/g, ' ').trim();
    if (seen.has(key) || seen.has(ckey)) { dropped.push({ model: it.model, title: it.title, reason: `duplicate of ${seen.get(key) || seen.get(ckey)}` }); continue; }
    seen.set(key, `${it.model}:${it.title}`); seen.set(ckey, `${it.model}:${it.title}`);
    it.id = 'arena-' + crypto.createHash('sha1').update(`${BATTLE}|${it.model}|${it.title}`).digest('hex').slice(0, 10);
    const p = prevById[it.id];
    it.status = (p && p.status) || 'pending';
    it.status_at = (p && p.status_at) || null;
    if (notes.length) it.note = notes.join('; ');
    items.push(it);
  }
  for (const m of Object.keys(perModel)) {
    perModel[m].kept = items.filter(i => i.model === m).length;
    perModel[m].dropped = dropped.filter(d => d.model === m).length;
  }
  const doc = { generated_at: new Date().toISOString(), source: `model-arena battle ${BATTLE}`, per_model: perModel, counts: { extracted: raw.length, kept: items.length, dropped: dropped.length }, dropped, items };
  const body = JSON.stringify(doc, null, 2);
  if (NAME.test(body)) { console.error('FAIL: naming rule hit in arena output'); process.exit(1); }
  fs.writeFileSync(OUT, body + '\n');
  console.log(`extracted ${raw.length}, kept ${items.length}, dropped ${dropped.length} -> data/arena-ideas.json`);
  for (const [m, v] of Object.entries(perModel)) console.log(`  ${m}: ${v.extracted} extracted via ${v.via}, ${v.kept} kept, ${v.dropped} dropped`);
  for (const d of dropped) console.log(`  dropped [${d.model}] ${d.title}: ${d.reason}`);
})().catch(e => { console.error(e); process.exit(2); });