← back to Wallco Ai
haiku 4.5 weekend defect-scan test scripts + monday summary
431fbf53ac31593ac133bb998285aa1687738180 · 2026-05-29 08:27:27 -0700 · Steve Abrams
build the runner (test-haiku-vision-weekend.js) and analyzer
(haiku-weekend-summary.js) for the weekend recall test against
documented Gemini blind spots (tone-on-tone seam breaks, fuzzy
heal bands at tile midlines, mural-misclass).
dtd verdict A (unanimous Claude+Codex, qwen errored): multi-label
rubric (OK/GHOST/FUZZY_SEAM/TONE_BREAK/MURAL_MISCLASS/VENDOR_LEAK/
BLEED/OTHER) gives per-category recall measurement vs binary or
identical-prompt head-to-head.
guards: budget cap ($10 default, auto-exit), 400ms rate limit,
dry-run default, resumable JSONL, auto-pause on 3 consecutive 5xx,
never touches is_published.
ground truth: bad_examples PG table weighted across vision-relevant
defect_tags + is_published=TRUE post-2026-05-27 (curator-approved)
+ live curator queue.
summary script computes per-tag recall on bads, FP rate on goods,
gemini-disagreement set (top-N each direction) for monday triage.
blocked on credits — $0 balance per /v1/messages 'credit balance
too low' error 2026-05-29.
Files touched
A scripts/haiku-weekend-summary.jsM scripts/test-haiku-vision-weekend.js
Diff
commit 431fbf53ac31593ac133bb998285aa1687738180
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Fri May 29 08:27:27 2026 -0700
haiku 4.5 weekend defect-scan test scripts + monday summary
build the runner (test-haiku-vision-weekend.js) and analyzer
(haiku-weekend-summary.js) for the weekend recall test against
documented Gemini blind spots (tone-on-tone seam breaks, fuzzy
heal bands at tile midlines, mural-misclass).
dtd verdict A (unanimous Claude+Codex, qwen errored): multi-label
rubric (OK/GHOST/FUZZY_SEAM/TONE_BREAK/MURAL_MISCLASS/VENDOR_LEAK/
BLEED/OTHER) gives per-category recall measurement vs binary or
identical-prompt head-to-head.
guards: budget cap ($10 default, auto-exit), 400ms rate limit,
dry-run default, resumable JSONL, auto-pause on 3 consecutive 5xx,
never touches is_published.
ground truth: bad_examples PG table weighted across vision-relevant
defect_tags + is_published=TRUE post-2026-05-27 (curator-approved)
+ live curator queue.
summary script computes per-tag recall on bads, FP rate on goods,
gemini-disagreement set (top-N each direction) for monday triage.
blocked on credits — $0 balance per /v1/messages 'credit balance
too low' error 2026-05-29.
---
scripts/haiku-weekend-summary.js | 249 +++++++++++++++++++++++++++++++++++
scripts/test-haiku-vision-weekend.js | 2 +-
2 files changed, 250 insertions(+), 1 deletion(-)
diff --git a/scripts/haiku-weekend-summary.js b/scripts/haiku-weekend-summary.js
new file mode 100644
index 0000000..40be0af
--- /dev/null
+++ b/scripts/haiku-weekend-summary.js
@@ -0,0 +1,249 @@
+#!/usr/bin/env node
+/**
+ * haiku-weekend-summary — analyze the JSONL output from
+ * test-haiku-vision-weekend.js into a Monday-morning verdict report.
+ *
+ * Computes:
+ * 1. Cost roll-up (total, per-bucket, per-verdict, per-1k-calls efficiency)
+ * 2. Per-defect-tag RECALL on the known-bads bucket — for each of
+ * ghost/bleed/repeat_break_lr/repeat_break_tb/auto_seam_break/
+ * manual_bad_region/edges_fail, what % did Haiku flag as non-OK
+ * (and with which label)
+ * 3. FALSE-POSITIVE rate on the known-goods bucket — what % did Haiku
+ * flag as a defect even though they were curator-approved
+ * 4. GEMINI DISAGREEMENT — cross-reference designs where Haiku verdict
+ * != Gemini's existing ghost-scan-flagged.jsonl verdict. Surfaces
+ * candidates for human triage (the most actionable output —
+ * either Haiku found something Gemini missed, or vice versa)
+ * 5. Live-queue verdict histogram + cost breakdown
+ *
+ * Writes a markdown report alongside the JSONL.
+ *
+ * Usage:
+ * node scripts/haiku-weekend-summary.js # picks newest jsonl
+ * node scripts/haiku-weekend-summary.js --in data/haiku-weekend-2026-05-29.jsonl
+ * node scripts/haiku-weekend-summary.js --top-disagreements 50 # how many to surface
+ */
+const fs = require('fs');
+const path = require('path');
+
+const argv = process.argv.slice(2);
+const ARG = (k, d) => { const i = argv.indexOf('--' + k); return i >= 0 ? argv[i + 1] : d; };
+
+const ROOT = path.join(__dirname, '..');
+const opt = {
+ in: ARG('in', null),
+ out: ARG('out', null),
+ topDisagree: parseInt(ARG('top-disagreements', '30'), 10),
+ geminiFlagged: ARG('gemini-flagged', path.join(ROOT, 'data', 'ghost-scan-flagged.jsonl')),
+};
+
+// Auto-pick newest jsonl if --in not given
+if (!opt.in) {
+ const dataDir = path.join(ROOT, 'data');
+ const candidates = fs.readdirSync(dataDir)
+ .filter(f => /^haiku-weekend-.*\.jsonl$/.test(f))
+ .map(f => ({ f, mtime: fs.statSync(path.join(dataDir, f)).mtimeMs }))
+ .sort((a, b) => b.mtime - a.mtime);
+ if (!candidates.length) { console.error('No haiku-weekend-*.jsonl found in data/'); process.exit(1); }
+ opt.in = path.join(dataDir, candidates[0].f);
+ console.log(`(auto-picked newest: ${candidates[0].f})`);
+}
+const inPath = path.isAbsolute(opt.in) ? opt.in : path.join(ROOT, opt.in);
+const outPath = opt.out
+ ? (path.isAbsolute(opt.out) ? opt.out : path.join(ROOT, opt.out))
+ : inPath.replace(/\.jsonl$/, '-summary.md');
+
+// ── load ──────────────────────────────────────────────────────────────
+function loadJsonl(p) {
+ if (!fs.existsSync(p)) return [];
+ return fs.readFileSync(p, 'utf8').split('\n')
+ .filter(Boolean)
+ .map(line => { try { return JSON.parse(line); } catch { return null; } })
+ .filter(Boolean);
+}
+
+const rows = loadJsonl(inPath);
+const goodRows = rows.filter(r => r.verdict); // exclude error rows + file_missing
+const errRows = rows.filter(r => r.error);
+console.log(`Loaded ${rows.length} rows (${goodRows.length} verdicts · ${errRows.length} errors/missing) from ${path.basename(inPath)}`);
+
+const gemini = loadJsonl(opt.geminiFlagged);
+const geminiFlaggedIds = new Set(gemini.map(r => r.id));
+console.log(`Loaded ${gemini.length} Gemini-flagged designs from ${path.basename(opt.geminiFlagged)}`);
+
+// ── computations ──────────────────────────────────────────────────────
+function pct(n, d) { return d === 0 ? '—' : `${(100*n/d).toFixed(1)}%`; }
+function bucket(rs, b) { return rs.filter(r => r.bucket === b); }
+
+// 1. cost roll-up
+const totalCost = goodRows.reduce((s,r) => s + (r.cost_usd || 0), 0);
+const costByBucket = {};
+const verdictHist = {};
+for (const r of goodRows) {
+ costByBucket[r.bucket] = (costByBucket[r.bucket] || 0) + (r.cost_usd || 0);
+ verdictHist[r.verdict] = (verdictHist[r.verdict] || 0) + 1;
+}
+
+// 2. per-defect-tag recall on the known-bads bucket
+const bads = bucket(goodRows, 'bad');
+const tagRecall = {}; // tag → { total, flagged_by_haiku, by_verdict: {LABEL: count} }
+for (const r of bads) {
+ for (const tag of (r.defect_tags || [])) {
+ if (!tagRecall[tag]) tagRecall[tag] = { total: 0, flagged: 0, byVerdict: {} };
+ tagRecall[tag].total++;
+ if (r.verdict !== 'OK' && r.verdict !== 'PARSE_FAIL') tagRecall[tag].flagged++;
+ tagRecall[tag].byVerdict[r.verdict] = (tagRecall[tag].byVerdict[r.verdict] || 0) + 1;
+ }
+}
+
+// 3. false-positive on goods
+const goods = bucket(goodRows, 'good');
+const goodFP = goods.filter(r => r.verdict !== 'OK' && r.verdict !== 'PARSE_FAIL');
+const goodFPByVerdict = {};
+for (const r of goodFP) goodFPByVerdict[r.verdict] = (goodFPByVerdict[r.verdict] || 0) + 1;
+
+// 4. Gemini disagreement
+// HaikuFlagsGeminiDidnt: Haiku verdict in {GHOST, FUZZY_SEAM, BLEED, MULTI_OPACITY} but NOT in Gemini flagged
+// GeminiFlagsHaikuDidnt: Was in Gemini flagged (ghost) but Haiku said OK
+const haikuFlaggedSet = new Set(goodRows.filter(r => ['GHOST','BLEED','FUZZY_SEAM'].includes(r.verdict)).map(r => r.id));
+const haikuFlagsGeminiDidnt = goodRows
+ .filter(r => ['GHOST','BLEED','FUZZY_SEAM'].includes(r.verdict) && !geminiFlaggedIds.has(r.id))
+ .sort((a, b) => (b.cost_usd || 0) - (a.cost_usd || 0))
+ .slice(0, opt.topDisagree);
+const geminiFlagsHaikuDidnt = goodRows
+ .filter(r => r.verdict === 'OK' && geminiFlaggedIds.has(r.id))
+ .slice(0, opt.topDisagree);
+
+// 5. live-queue verdict histogram
+const live = bucket(goodRows, 'live');
+const liveVerdict = {};
+for (const r of live) liveVerdict[r.verdict] = (liveVerdict[r.verdict] || 0) + 1;
+
+// ── render ────────────────────────────────────────────────────────────
+const lines = [];
+const push = s => lines.push(s);
+
+push(`# Haiku 4.5 weekend defect-scan summary`);
+push(`*Generated ${new Date().toISOString()} · source ${path.basename(inPath)}*`);
+push('');
+
+push(`## TL;DR`);
+push(`- **${goodRows.length} verdicts** processed (${errRows.length} errors/missing).`);
+push(`- **Total spend: $${totalCost.toFixed(4)}** at ~$${goodRows.length ? (totalCost/goodRows.length).toFixed(5) : '?'}/call.`);
+push(`- **Known-bads caught:** ${pct(bads.filter(r => r.verdict !== 'OK' && r.verdict !== 'PARSE_FAIL').length, bads.length)} (${bads.length} bads in sample).`);
+push(`- **Known-goods false-positive rate:** ${pct(goodFP.length, goods.length)} (${goods.length} goods in sample).`);
+push(`- **Gemini disagreements:** ${haikuFlagsGeminiDidnt.length} where Haiku flagged but Gemini did not; ${geminiFlagsHaikuDidnt.length} where Gemini flagged but Haiku said OK.`);
+push('');
+
+push(`## Cost roll-up`);
+push(`| bucket | rows | cost ($) | per-call avg |`);
+push(`|---|---:|---:|---:|`);
+for (const b of ['bad','good','live']) {
+ const rs = bucket(goodRows, b);
+ const c = costByBucket[b] || 0;
+ push(`| ${b} | ${rs.length} | ${c.toFixed(4)} | ${rs.length ? '$'+(c/rs.length).toFixed(5) : '—'} |`);
+}
+push(`| **total** | **${goodRows.length}** | **${totalCost.toFixed(4)}** | **${goodRows.length ? '$'+(totalCost/goodRows.length).toFixed(5) : '—'}** |`);
+push('');
+
+push(`## Per-defect-tag recall (known-bads bucket)`);
+push(`How often Haiku assigned a non-OK label to a row Steve had already curated as a known-bad. Sorted by sample size.`);
+push('');
+push(`| defect_tag | n | flagged | recall | top Haiku labels |`);
+push(`|---|---:|---:|---:|---|`);
+const tagsSorted = Object.entries(tagRecall).sort((a,b) => b[1].total - a[1].total);
+for (const [tag, t] of tagsSorted) {
+ const top = Object.entries(t.byVerdict).sort((a,b) => b[1]-a[1]).slice(0, 4)
+ .map(([v,c]) => `${v}:${c}`).join(' ');
+ push(`| \`${tag}\` | ${t.total} | ${t.flagged} | ${pct(t.flagged, t.total)} | ${top} |`);
+}
+push('');
+push(`> **Gold blind-spot tags** are \`repeat_break_lr\`, \`repeat_break_tb\`, \`auto_seam_break\`, \`manual_bad_region\`. Recall on these is the headline number — that is where Gemini was documented blind.`);
+push('');
+
+push(`## False-positive rate on known-goods`);
+push(`These are designs that passed the curator and are \`is_published=TRUE\`. Anything Haiku flagged here is either (a) a Haiku FP, (b) a defect the curator missed, or (c) drift since publish.`);
+push('');
+push(`| Haiku verdict | count | % of goods |`);
+push(`|---|---:|---:|`);
+for (const [v, c] of Object.entries(goodFPByVerdict).sort((a,b) => b[1]-a[1])) {
+ push(`| ${v} | ${c} | ${pct(c, goods.length)} |`);
+}
+push(`| **total flagged** | **${goodFP.length}** | **${pct(goodFP.length, goods.length)}** |`);
+push('');
+
+push(`## Verdict histogram`);
+push(`Distribution of Haiku verdicts across the full sample.`);
+push('');
+push(`| verdict | count | % |`);
+push(`|---|---:|---:|`);
+for (const [v, c] of Object.entries(verdictHist).sort((a,b) => b[1]-a[1])) {
+ push(`| ${v} | ${c} | ${pct(c, goodRows.length)} |`);
+}
+push('');
+
+if (live.length) {
+ push(`## Live-queue verdicts (${live.length} unjudged rows)`);
+ push(`What Haiku said about designs that haven't been curated yet. These are the rows most actionable for the cactus-curator queue.`);
+ push('');
+ push(`| verdict | count |`);
+ push(`|---|---:|`);
+ for (const [v, c] of Object.entries(liveVerdict).sort((a,b) => b[1]-a[1])) {
+ push(`| ${v} | ${c} |`);
+ }
+ push('');
+}
+
+push(`## Gemini disagreements — Haiku flagged, Gemini did not (top ${haikuFlagsGeminiDidnt.length})`);
+push(`These are candidates for human triage. If they are real defects, Gemini is missing them.`);
+push('');
+push(`| id | bucket | category | Haiku verdict | reason |`);
+push(`|---|---|---|---|---|`);
+for (const r of haikuFlagsGeminiDidnt) {
+ const reason = (r.reason || '').replace(/\|/g, '\\|').slice(0, 100);
+ push(`| ${r.id} | ${r.bucket} | ${r.category || '—'} | ${r.verdict} | ${reason} |`);
+}
+push('');
+
+push(`## Gemini disagreements — Gemini flagged, Haiku said OK (top ${geminiFlagsHaikuDidnt.length})`);
+push(`If Haiku is right, these are Gemini false-positives sitting in \`ghost-scan-flagged.jsonl\` that the publish gate has been holding back.`);
+push('');
+push(`| id | bucket | category | Haiku reason |`);
+push(`|---|---|---|---|`);
+for (const r of geminiFlagsHaikuDidnt) {
+ const reason = (r.reason || '').replace(/\|/g, '\\|').slice(0, 100);
+ push(`| ${r.id} | ${r.bucket} | ${r.category || '—'} | ${reason} |`);
+}
+push('');
+
+push(`## Errors`);
+if (errRows.length === 0) {
+ push(`No errors.`);
+} else {
+ push(`${errRows.length} rows had errors. Sample:`);
+ push('');
+ const errHist = {};
+ for (const r of errRows) {
+ const k = r.error || 'unknown';
+ errHist[k] = (errHist[k] || 0) + 1;
+ }
+ push(`| error | count |`);
+ push(`|---|---:|`);
+ for (const [k, v] of Object.entries(errHist).sort((a,b) => b[1]-a[1])) {
+ push(`| ${k.slice(0, 80)} | ${v} |`);
+ }
+}
+push('');
+
+push(`## Next actions`);
+push(`1. **Spot-check 5-10 rows from each Gemini-disagreement table** — eyeball whether Haiku is right.`);
+push(`2. **If Haiku catches the gold blind-spot tags at >60% recall**, wire it in as a parallel publish-gate verdict (cheap second opinion at ~$0.002/design).`);
+push(`3. **If Haiku FP rate on goods is < 5%**, consider replacing the Gemini ghost scanner outright.`);
+push(`4. **If neither**, the test paid for itself in surfacing the disagreement set — feed that into \`bad_examples\` as labeled training data for a smaller-than-Haiku detector.`);
+push('');
+
+fs.writeFileSync(outPath, lines.join('\n'));
+console.log(`\nWrote ${outPath}`);
+console.log(` ${goodRows.length} verdicts · $${totalCost.toFixed(4)} total spend`);
+console.log(` ${haikuFlagsGeminiDidnt.length} Haiku-flags-Gemini-missed · ${geminiFlagsHaikuDidnt.length} Gemini-flags-Haiku-cleared`);
diff --git a/scripts/test-haiku-vision-weekend.js b/scripts/test-haiku-vision-weekend.js
index e31a16c..100dbd5 100644
--- a/scripts/test-haiku-vision-weekend.js
+++ b/scripts/test-haiku-vision-weekend.js
@@ -45,7 +45,7 @@ const opt = {
live: parseInt(ARG('live', '200'), 10),
budgetCap: parseFloat(ARG('budget-cap-usd', '10')),
rateLimitMs: parseInt(ARG('rate-limit-ms', '400'), 10),
- model: ARG('model', 'claude-opus-4-8'),
+ model: ARG('model', 'claude-haiku-4-5'),
out: ARG('out', `data/haiku-weekend-${new Date().toISOString().slice(0,10)}.jsonl`),
goodsSince: ARG('goods-since', '2026-05-27'), // post curator-cleanup window
};
← 63833d5 security: remove vestigial admin:DWSecure2024! basic-auth st
·
back to Wallco Ai
·
Add rollback-photoreal-rooms.py: regenerate living_room mock d335cbc →