`demo/headtohead.mjs`

133 lines
#!/usr/bin/env node
// Head-to-head: Hermes-3-8B vs Qwen3:14b on a real DW task.
// Prompt: 5 wallpaper concept briefs for a "Costa Brava" coastal-Spanish-luxury collection.
// Strict JSON output → measure validity, latency, palette quality.

import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const OUT = path.join(__dirname, 'output');
fs.mkdirSync(OUT, { recursive: true });

const HOST = process.env.OLLAMA_HOST || 'http://127.0.0.1:11434';

const SYSTEM = `You are a senior wallpaper designer at Designer Wallcoverings.
You speak in tight, editorial copy — the way a Vogue feature writes about an interior.
You return STRICT JSON only. No prose, no markdown fences, no commentary.
Schema:
{
  "collection": "string",
  "concepts": [
    {
      "name": "string (2-3 words, evocative)",
      "hero_hex": "#RRGGBB",
      "palette": ["#RRGGBB", "#RRGGBB", "#RRGGBB"],
      "motif": "string (one sentence)",
      "room": "string (single room type)",
      "tagline": "string (max 8 words)",
      "tier": "Studio | Atelier | Bespoke"
    }
  ]
}`;

const USER = `Brief: Designer Wallcoverings is launching the COSTA BRAVA collection — coastal Spanish luxury, sun-bleached terracotta and Mediterranean blue, hand-block prints reinterpreted for 2026 interiors. Target client: hospitality + high-end residential.

Generate exactly 5 distinct wallpaper concepts. Each must have a unique hero_hex and a palette that genuinely complements it. Names should evoke place, not be generic. Taglines must SELL — not describe.

Return strict JSON only.`;

async function callOllama(model, system, user) {
  const t0 = Date.now();
  const r = await fetch(`${HOST}/api/chat`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({
      model,
      stream: false,
      format: 'json',
      options: { temperature: 0.7, num_ctx: 4096, num_predict: 1500 },
      messages: [
        { role: 'system', content: system },
        { role: 'user', content: user },
      ],
    }),
  });
  const j = await r.json();
  const ms = Date.now() - t0;
  return {
    model,
    ms,
    raw: j.message?.content ?? '',
    eval_count: j.eval_count,
    prompt_eval_count: j.prompt_eval_count,
  };
}

function tryParse(raw) {
  try { return { ok: true, data: JSON.parse(raw) }; }
  catch (e) {
    const m = raw.match(/\{[\s\S]*\}/);
    if (m) { try { return { ok: true, data: JSON.parse(m[0]), salvaged: true }; } catch {} }
    return { ok: false, error: e.message };
  }
}

function validate(data) {
  const issues = [];
  if (!data?.concepts || !Array.isArray(data.concepts)) {
    return { issues: ['no concepts array'], valid: 0, total: 0 };
  }
  const total = data.concepts.length;
  let valid = 0;
  for (const c of data.concepts) {
    const ok =
      typeof c.name === 'string' && c.name.trim().length > 0 &&
      /^#[0-9a-fA-F]{6}$/.test(c.hero_hex || '') &&
      Array.isArray(c.palette) && c.palette.length >= 3 &&
      c.palette.every(h => /^#[0-9a-fA-F]{6}$/.test(h)) &&
      typeof c.motif === 'string' &&
      typeof c.tagline === 'string' &&
      (c.tagline.split(/\s+/).filter(Boolean).length <= 8) &&
      ['Studio', 'Atelier', 'Bespoke'].includes(c.tier);
    if (ok) valid++; else issues.push(`bad concept: ${JSON.stringify(c).slice(0, 80)}…`);
  }
  return { issues, valid, total };
}

console.log('→ pinging both models in parallel…');
const [hermes, qwen] = await Promise.all([
  callOllama('hermes3:8b', SYSTEM, USER),
  callOllama('qwen3:14b', SYSTEM, USER),
]);

for (const r of [hermes, qwen]) {
  const slug = r.model.replace(/[^a-z0-9]+/gi, '_');
  fs.writeFileSync(path.join(OUT, `${slug}.raw.txt`), r.raw);
  const p = tryParse(r.raw);
  const v = p.ok ? validate(p.data) : { issues: [p.error], valid: 0, total: 0 };
  const summary = {
    model: r.model,
    latency_ms: r.ms,
    tokens_out: r.eval_count,
    tokens_in: r.prompt_eval_count,
    parsed_ok: p.ok,
    salvaged: !!p.salvaged,
    valid_concepts: v.valid,
    total_concepts: v.total,
    issues: v.issues,
    data: p.ok ? p.data : null,
  };
  fs.writeFileSync(path.join(OUT, `${slug}.json`), JSON.stringify(summary, null, 2));
  console.log(`\n[${r.model}]  ${r.ms}ms  parsed=${p.ok}  valid=${v.valid}/${v.total}  tokens=${r.eval_count}`);
  if (p.ok && p.data?.concepts) {
    for (const c of p.data.concepts) {
      console.log(`   ${c.hero_hex || '------'}  ${(c.name || '?').padEnd(22)}  ${c.tagline || ''}`);
    }
  }
}

console.log('\n✓ outputs written to demo/output/');
console.log('  hermes3_8b.json     qwen3_14b.json');