← back to Wallco Ai
Update Claude model IDs to claude-opus-4-8 (Opus 4.8 upgrade)
692c3ba64df1cbf21b3f78d0714b6d452f2c56b2 · 2026-05-29 08:16:37 -0700 · Steve
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
A scripts/test-haiku-vision-weekend.js
Diff
commit 692c3ba64df1cbf21b3f78d0714b6d452f2c56b2
Author: Steve <steve@designerwallcoverings.com>
Date: Fri May 29 08:16:37 2026 -0700
Update Claude model IDs to claude-opus-4-8 (Opus 4.8 upgrade)
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
scripts/test-haiku-vision-weekend.js | 336 +++++++++++++++++++++++++++++++++++
1 file changed, 336 insertions(+)
diff --git a/scripts/test-haiku-vision-weekend.js b/scripts/test-haiku-vision-weekend.js
new file mode 100644
index 0000000..e31a16c
--- /dev/null
+++ b/scripts/test-haiku-vision-weekend.js
@@ -0,0 +1,336 @@
+#!/usr/bin/env node
+/**
+ * test-haiku-vision-weekend — vision-classify designs via Claude Haiku 4.5
+ * to measure recall against documented Gemini blind spots.
+ *
+ * Why this exists (Steve 2026-05-29): Gemini ghost-detector catches ghost-layer
+ * + bleed reliably but is blind to (a) fuzzy heal blur bands across tile midlines
+ * (`feedback_wallco_heal_blur_band_edges_agent_blind`, 4,022/4,071 missed in one
+ * batch) and (b) tone-on-tone broken L/R/T-B repeats where motifs slice at the
+ * seam (`feedback_wallco_tone_on_tone_repeat_break_needs_vision`, 0% pixel-diff
+ * recall). This script tests whether Haiku 4.5 vision catches them at $0.002/call
+ * vs Gemini's $0.0005/call.
+ *
+ * GROUND TRUTH:
+ * - known-bads: bad_examples table, weighted across defect_tags
+ * - known-goods: is_published=TRUE rows post-2026-05-27 (curator-approved)
+ * - live queue: latest unjudged designs (open-ended judgment)
+ *
+ * HARD GUARDS:
+ * - --budget-cap-usd=10 (auto-exit on cumulative cross)
+ * - --rate-limit-ms=400 (~150 RPM ceiling — well under Haiku Tier 1 limit)
+ * - --dry-run is the default; pass --commit to actually fire
+ * - DOES NOT touch is_published; classification only
+ * - Resumable: skips IDs already in the JSONL output
+ * - Auto-pause on 3 consecutive 5xx
+ *
+ * Usage:
+ * node scripts/test-haiku-vision-weekend.js # dry-run preview
+ * node scripts/test-haiku-vision-weekend.js --commit # fire (1200 calls)
+ * node scripts/test-haiku-vision-weekend.js --commit --bads 100 --goods 100 --live 0 # smoke
+ * node scripts/test-haiku-vision-weekend.js --commit --budget-cap-usd 5
+ */
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+
+// ── args ──────────────────────────────────────────────────────────────
+const argv = process.argv.slice(2);
+const ARG = (k, d) => { const i = argv.indexOf('--' + k); return i >= 0 ? argv[i + 1] : d; };
+const HAS = k => argv.includes('--' + k);
+const opt = {
+ commit: HAS('commit'),
+ bads: parseInt(ARG('bads', '500'), 10),
+ goods: parseInt(ARG('goods', '500'), 10),
+ live: parseInt(ARG('live', '200'), 10),
+ budgetCap: parseFloat(ARG('budget-cap-usd', '10')),
+ rateLimitMs: parseInt(ARG('rate-limit-ms', '400'), 10),
+ model: ARG('model', 'claude-opus-4-8'),
+ out: ARG('out', `data/haiku-weekend-${new Date().toISOString().slice(0,10)}.jsonl`),
+ goodsSince: ARG('goods-since', '2026-05-27'), // post curator-cleanup window
+};
+
+// ── pricing (Haiku 4.5 standard tier) ────────────────────────────────
+const PRICE = { input: 1.00 / 1_000_000, output: 5.00 / 1_000_000 };
+
+// ── env ───────────────────────────────────────────────────────────────
+function loadApiKey() {
+ if (process.env.ANTHROPIC_API_KEY) return process.env.ANTHROPIC_API_KEY;
+ const candidates = [
+ path.join(process.env.HOME, 'Desktop', 'site-factory.env'),
+ path.join(process.env.HOME, 'Projects', 'secrets-manager', '.env'),
+ path.join(__dirname, '..', '.env'),
+ ];
+ for (const f of candidates) {
+ if (!fs.existsSync(f)) continue;
+ const m = fs.readFileSync(f, 'utf8').match(/^ANTHROPIC_API_KEY=(.+)$/m);
+ if (m) return m[1].trim().replace(/^["']|["']$/g, '');
+ }
+ throw new Error('ANTHROPIC_API_KEY missing (checked env + ~/Desktop/site-factory.env + ~/Projects/secrets-manager/.env)');
+}
+
+// ── PG helpers ────────────────────────────────────────────────────────
+function psql(sql) {
+ const r = spawnSync('psql', ['dw_unified', '-At', '-q'], { input: sql, encoding: 'utf8' });
+ if (r.status !== 0) throw new Error(`psql failed: ${r.stderr}`);
+ return r.stdout.split('\n').filter(Boolean).map(line => line.split('|'));
+}
+
+// Sample distribution across vision-relevant defect tags (Steve's note:
+// the gold blind-spot rows — repeat_break_lr/tb + auto_seam_break — are
+// rare, so we include them ALL even if `--bads` is set lower).
+function fetchBads(target) {
+ const goldTags = ['repeat_break_lr', 'repeat_break_tb', 'auto_seam_break', 'manual_bad_region'];
+ const bulkTags = ['ghost', 'bleed', 'edges_fail'];
+
+ // ALL gold-tag rows w/ a local_path
+ const goldSql = `
+ SELECT DISTINCT b.design_id, b.category, array_to_string(b.defect_tags,','), d.kind, b.local_path
+ FROM bad_examples b
+ JOIN spoon_all_designs d ON d.id = b.design_id
+ WHERE b.local_path IS NOT NULL
+ AND b.defect_tags && ARRAY['${goldTags.join("','")}']::text[]`;
+ const gold = psql(goldSql);
+
+ const goldIds = new Set(gold.map(r => r[0]));
+ const remaining = Math.max(0, target - gold.length);
+
+ // Weighted bulk: split across ghost/bleed/edges_fail roughly equally
+ const perTag = Math.ceil(remaining / bulkTags.length);
+ let bulk = [];
+ for (const tag of bulkTags) {
+ const sql = `
+ SELECT DISTINCT b.design_id, b.category, array_to_string(b.defect_tags,','), d.kind, b.local_path
+ FROM bad_examples b
+ JOIN spoon_all_designs d ON d.id = b.design_id
+ WHERE b.local_path IS NOT NULL
+ AND '${tag}' = ANY(b.defect_tags)
+ ORDER BY random()
+ LIMIT ${perTag * 2}`;
+ const rows = psql(sql).filter(r => !goldIds.has(r[0]));
+ bulk = bulk.concat(rows.slice(0, perTag));
+ rows.slice(0, perTag).forEach(r => goldIds.add(r[0]));
+ }
+
+ return [...gold, ...bulk.slice(0, remaining)].map(r => ({
+ id: parseInt(r[0], 10), category: r[1] || null, defect_tags: r[2] ? r[2].split(',') : [],
+ kind: r[3] || null, local_path: r[4], bucket: 'bad',
+ }));
+}
+
+function fetchGoods(target) {
+ const sql = `
+ SELECT id, category, kind, local_path
+ FROM spoon_all_designs
+ WHERE is_published = TRUE
+ AND local_path IS NOT NULL
+ AND created_at >= '${opt.goodsSince}'::date
+ AND id NOT IN (SELECT DISTINCT design_id FROM bad_examples)
+ ORDER BY random()
+ LIMIT ${target}`;
+ return psql(sql).map(r => ({
+ id: parseInt(r[0], 10), category: r[1] || null, kind: r[2] || null,
+ local_path: r[3], defect_tags: [], bucket: 'good',
+ }));
+}
+
+function fetchLive(target) {
+ // Latest curator queue — neither flagged nor explicitly approved.
+ const sql = `
+ SELECT id, category, kind, local_path
+ FROM spoon_all_designs
+ WHERE local_path IS NOT NULL
+ AND (is_published IS NULL OR is_published = FALSE)
+ AND needs_fixing_at IS NULL
+ AND id NOT IN (SELECT DISTINCT design_id FROM bad_examples)
+ ORDER BY created_at DESC
+ LIMIT ${target}`;
+ return psql(sql).map(r => ({
+ id: parseInt(r[0], 10), category: r[1] || null, kind: r[2] || null,
+ local_path: r[3], defect_tags: [], bucket: 'live',
+ }));
+}
+
+// ── prompt ────────────────────────────────────────────────────────────
+function buildPrompt(d) {
+ const isMural = (d.kind === 'mural_panel' || d.kind === 'mural');
+ return `You are auditing a wallpaper design for production defects. The design ID is #${d.id}, category "${d.category || '(unknown)'}", kind "${d.kind || '(unknown)'}" (${isMural ? 'MURAL — single image, not tileable' : 'TILE — should repeat seamlessly L↔R and T↔B'}).
+
+Classify the image as EXACTLY ONE label, then on a new line give ONE sentence (max 25 words) explaining why.
+
+Labels:
+- OK — clean, no production defects.
+- GHOST — translucent / partial-opacity layers, ghosted shapes behind the main motif, or visible multi-opacity print artifacts. Solid screen-print only is the standard.
+- FUZZY_SEAM — a blurred or smeared horizontal/vertical band crossing the middle of the tile (~12 px wide), or fuzzy heal artifacts at the edges. Distinct from a sharp clean seam.
+- TONE_BREAK — ${isMural ? 'N/A for murals — never use this label' : 'the pattern is broken at the L/R or T/B edges; motifs are sliced at the wrap and do not match up across the seam (e.g. half a cactus on the right edge, no matching half on the left). This is subtle on tone-on-tone designs.'}
+- MURAL_MISCLASS — ${isMural ? 'this is correctly a mural, so this label does NOT apply' : 'this image is actually a single-subject mural / scenic, NOT a repeating tile, despite being categorized as a tile.'}
+- VENDOR_LEAK — visible text containing a vendor brand name (e.g. "Schumacher", "Brunschwig", "Lee Jofa", "Thibaut", "Kravet", "Arte", "Cole & Son", "Wolf-Gordon").
+- BLEED — color bleeds outside its motif boundary or smears at edges.
+- OTHER — some other quality issue; specify briefly in the reason line.
+
+Respond with JUST the label on line 1, JUST one sentence on line 2. No preamble, no markdown.`;
+}
+
+// ── Anthropic API call ────────────────────────────────────────────────
+async function callHaiku(imagePath, prompt, apiKey) {
+ const imgB64 = fs.readFileSync(imagePath).toString('base64');
+ const ext = path.extname(imagePath).toLowerCase().replace('.', '') || 'png';
+ const mediaType = ext === 'jpg' || ext === 'jpeg' ? 'image/jpeg' : `image/${ext}`;
+
+ const body = {
+ model: opt.model,
+ max_tokens: 100,
+ messages: [{
+ role: 'user',
+ content: [
+ { type: 'image', source: { type: 'base64', media_type: mediaType, data: imgB64 } },
+ { type: 'text', text: prompt },
+ ],
+ }],
+ };
+
+ const r = await fetch('https://api.anthropic.com/v1/messages', {
+ method: 'POST',
+ headers: {
+ 'x-api-key': apiKey,
+ 'anthropic-version': '2023-06-01',
+ 'content-type': 'application/json',
+ },
+ body: JSON.stringify(body),
+ });
+ const j = await r.json();
+ if (!r.ok) {
+ const err = new Error(j.error?.message || `HTTP ${r.status}`);
+ err.status = r.status;
+ err.payload = j;
+ throw err;
+ }
+ return j;
+}
+
+// Parse "LABEL\nreason" — be forgiving about formatting drift
+function parseVerdict(content) {
+ const text = (content || []).map(b => b.text || '').join('\n').trim();
+ const lines = text.split('\n').map(s => s.trim()).filter(Boolean);
+ if (!lines.length) return { label: 'PARSE_FAIL', reason: '(empty)', raw: text };
+ const first = lines[0].replace(/[^A-Z_]/g, '').toUpperCase();
+ const valid = ['OK','GHOST','FUZZY_SEAM','TONE_BREAK','MURAL_MISCLASS','VENDOR_LEAK','BLEED','OTHER'];
+ const label = valid.includes(first) ? first : 'PARSE_FAIL';
+ return { label, reason: lines.slice(1).join(' ').slice(0, 240), raw: text };
+}
+
+// ── main ──────────────────────────────────────────────────────────────
+async function main() {
+ console.log(`\n══ Haiku 4.5 vision weekend test ══`);
+ console.log(` Model: ${opt.model}`);
+ console.log(` Budget cap: $${opt.budgetCap.toFixed(2)}`);
+ console.log(` Rate limit: ${opt.rateLimitMs}ms between calls`);
+ console.log(` Output: ${opt.out}`);
+ console.log(` Mode: ${opt.commit ? '🔴 COMMIT' : '🟡 DRY-RUN'}\n`);
+
+ const apiKey = loadApiKey();
+
+ console.log(`Sampling…`);
+ const bads = fetchBads(opt.bads);
+ const goods = fetchGoods(opt.goods);
+ const live = fetchLive(opt.live);
+ console.log(` bad_examples: ${bads.length} (target ${opt.bads}; gold blind-spot tags always included)`);
+ console.log(` known-goods: ${goods.length} (target ${opt.goods}; post-${opt.goodsSince})`);
+ console.log(` live-queue: ${live.length} (target ${opt.live})`);
+
+ const all = [...bads, ...goods, ...live];
+ const estimatedCost = all.length * 0.002; // ~$0.002 per Haiku 4.5 vision call
+ console.log(` total: ${all.length}`);
+ console.log(` estimated cost: $${estimatedCost.toFixed(2)} (well under $${opt.budgetCap} cap)\n`);
+
+ // Resumability — skip IDs already in the output
+ const outPath = path.join(__dirname, '..', opt.out);
+ fs.mkdirSync(path.dirname(outPath), { recursive: true });
+ const seenIds = new Set();
+ let cumulativeCost = 0;
+ if (fs.existsSync(outPath)) {
+ for (const line of fs.readFileSync(outPath, 'utf8').split('\n')) {
+ if (!line.trim()) continue;
+ try {
+ const row = JSON.parse(line);
+ seenIds.add(row.id);
+ cumulativeCost += row.cost_usd || 0;
+ } catch {}
+ }
+ if (seenIds.size) {
+ console.log(`Resumable: ${seenIds.size} IDs already in ${opt.out} (cumulative $${cumulativeCost.toFixed(4)} so far)\n`);
+ }
+ }
+ const queue = all.filter(d => !seenIds.has(d.id));
+ console.log(`Queue after resume-filter: ${queue.length}\n`);
+
+ if (!opt.commit) {
+ console.log(`🟡 DRY-RUN. Pass --commit to actually fire.\n`);
+ console.log(`Sample of first 5 from queue:`);
+ queue.slice(0, 5).forEach(d => console.log(` #${d.id} ${d.bucket}/${d.kind}/${d.category} → ${d.local_path}`));
+ return;
+ }
+
+ // ── fire ──
+ const startTs = Date.now();
+ let ok = 0, err = 0, consecutiveErr = 0;
+ const out = fs.createWriteStream(outPath, { flags: 'a' });
+
+ for (let i = 0; i < queue.length; i++) {
+ if (cumulativeCost >= opt.budgetCap) {
+ console.log(`\n⏹ BUDGET CAP REACHED ($${cumulativeCost.toFixed(4)} ≥ $${opt.budgetCap}). Stopping.`);
+ break;
+ }
+ const d = queue[i];
+
+ // Skip if the local file vanished (orphan)
+ if (!fs.existsSync(d.local_path)) {
+ process.stdout.write(` (${i+1}/${queue.length}) #${d.id} ${d.bucket} — ⚠ orphan (file missing)\n`);
+ out.write(JSON.stringify({ id: d.id, bucket: d.bucket, ts: new Date().toISOString(), error: 'file_missing', cost_usd: 0, cumulative_usd: cumulativeCost }) + '\n');
+ continue;
+ }
+
+ process.stdout.write(` (${i+1}/${queue.length}) #${d.id} ${d.bucket}/${d.kind || '?'} `);
+ try {
+ const prompt = buildPrompt(d);
+ const resp = await callHaiku(d.local_path, prompt, apiKey);
+ const verdict = parseVerdict(resp.content);
+ const usage = resp.usage || {};
+ const cost = (usage.input_tokens || 0) * PRICE.input + (usage.output_tokens || 0) * PRICE.output;
+ cumulativeCost += cost;
+ consecutiveErr = 0;
+ ok++;
+ const row = {
+ id: d.id, bucket: d.bucket, category: d.category, kind: d.kind,
+ defect_tags: d.defect_tags, ts: new Date().toISOString(),
+ verdict: verdict.label, reason: verdict.reason,
+ tokens_in: usage.input_tokens, tokens_out: usage.output_tokens,
+ cost_usd: cost, cumulative_usd: cumulativeCost,
+ };
+ out.write(JSON.stringify(row) + '\n');
+ process.stdout.write(`→ ${verdict.label} (in=${usage.input_tokens} out=${usage.output_tokens} $${cost.toFixed(5)} cum=$${cumulativeCost.toFixed(4)})\n`);
+ } catch (e) {
+ err++; consecutiveErr++;
+ const errRow = { id: d.id, bucket: d.bucket, ts: new Date().toISOString(), error: e.message, status: e.status || null, cumulative_usd: cumulativeCost };
+ out.write(JSON.stringify(errRow) + '\n');
+ process.stdout.write(`✗ ${e.status || ''} ${e.message.slice(0, 80)}\n`);
+ if (consecutiveErr >= 3) {
+ console.log(`\n⚠ 3 consecutive errors — pausing 60s to let upstream recover.`);
+ await new Promise(r => setTimeout(r, 60_000));
+ consecutiveErr = 0;
+ }
+ }
+ if (i < queue.length - 1) await new Promise(r => setTimeout(r, opt.rateLimitMs));
+ }
+
+ out.end();
+ const elapsedMin = ((Date.now() - startTs) / 60_000).toFixed(1);
+ console.log(`\n══ done ══`);
+ console.log(` Processed: ${ok} ok · ${err} errors`);
+ console.log(` Spent: $${cumulativeCost.toFixed(4)} of $${opt.budgetCap} budget`);
+ console.log(` Wall time: ${elapsedMin} min`);
+ console.log(` Output: ${opt.out}`);
+ console.log(`\nRun the summary: node scripts/haiku-weekend-summary.js --in ${opt.out}\n`);
+}
+
+main().catch(e => { console.error('fatal:', e.message); process.exit(1); });
← 84a88cf docs: Digital Downloads app delivery runbook (chosen over we
·
back to Wallco Ai
·
security: remove vestigial admin:DWSecure2024! basic-auth st 63833d5 →