← back to Dw Dead Image Recovery
TK-11048: verify Versa dead-image recovery (13 negative-case tests, 40/40 independent liveness, guarded-SQL scan); fix ESM require() crash in --apply path + add direct-run guard for testability
10b13e47421a9d0934efc6a5138f851bdaaeb790 · 2026-09-02 13:53:45 -0700 · Steve Abrams
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01R9pqyFahiGfXofjzY2q6Hs
Files touched
M scripts/recover-versa.mjsA scripts/test-recover-versa.mjsA verification/e2e-proof.json
Diff
commit 10b13e47421a9d0934efc6a5138f851bdaaeb790
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Sep 2 13:53:45 2026 -0700
TK-11048: verify Versa dead-image recovery (13 negative-case tests, 40/40 independent liveness, guarded-SQL scan); fix ESM require() crash in --apply path + add direct-run guard for testability
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01R9pqyFahiGfXofjzY2q6Hs
---
scripts/recover-versa.mjs | 115 ++++++++++++++++++++++-------------------
scripts/test-recover-versa.mjs | 63 ++++++++++++++++++++++
verification/e2e-proof.json | 36 +++++++++++++
3 files changed, 161 insertions(+), 53 deletions(-)
diff --git a/scripts/recover-versa.mjs b/scripts/recover-versa.mjs
index fa96327..e848710 100644
--- a/scripts/recover-versa.mjs
+++ b/scripts/recover-versa.mjs
@@ -26,8 +26,8 @@
// node recover-versa.mjs --limit 60 # dry-run, bounded sample
// node recover-versa.mjs --apply # GATED write (Steve-approved only)
import { execFileSync } from 'node:child_process';
-import { writeFileSync, mkdirSync, appendFileSync } from 'node:fs';
-import { fileURLToPath } from 'node:url';
+import { writeFileSync, mkdirSync, appendFileSync, readFileSync } from 'node:fs';
+import { fileURLToPath, pathToFileURL } from 'node:url';
import { dirname, join } from 'node:path';
const HERE = dirname(fileURLToPath(import.meta.url));
@@ -49,7 +49,7 @@ function psql(sql) {
return execFileSync('psql', ['-h', '/tmp', '-d', 'dw_unified', '-Atc', sql],
{ encoding: 'utf8', timeout: 120000, maxBuffer: MAXBUF }).trim();
}
-const isDataUri = (u) => /^\s*data:|;base64,/i.test(String(u || ''));
+export const isDataUri = (u) => /^\s*data:|;base64,/i.test(String(u || ''));
async function withTimeout(fn) {
const ac = new AbortController(); const t = setTimeout(() => ac.abort(), TIMEOUT);
try { return await fn(ac.signal); } finally { clearTimeout(t); }
@@ -60,7 +60,7 @@ async function getText(url) {
return res.text();
}
// verify a candidate is a real, live image
-async function isLiveImage(url) {
+export async function isLiveImage(url) {
try {
let res = await withTimeout((s) => fetch(url, { method: 'HEAD', redirect: 'follow', signal: s, headers: UA }));
let ct = (res.headers.get('content-type') || '').toLowerCase();
@@ -77,7 +77,7 @@ async function isLiveImage(url) {
// Extract the best real image URL for a given SKU from the product-page HTML.
// Strategy: prefer a processed (larger) srcset entry whose filename contains the
// SKU core (AVFnn-nnn); else the /Thumbnails/ path; else the largest srcset img.
-function extractImage(html, sku) {
+export function extractImage(html, sku) {
const core = (sku.match(/^[A-Za-z]+\d+-\d+/) || [sku])[0]; // AVF25-834
const abs = (p) => (p.startsWith('http') ? p : HOST + p);
const allImgs = [...html.matchAll(/\/fileadmin\/[^"'\s]+?\.(?:jpg|jpeg|png|webp)/gi)].map((m) => m[0]);
@@ -96,56 +96,65 @@ function extractImage(html, sku) {
return url;
}
-const where = `et.catalog_table='versa_catalog' AND et.phase3_ai_at IS NULL AND et.last_error='Gemini returned no data' AND coalesce(c.product_url,'')<>''`;
-const rows = psql(
- `SELECT c.mfr_sku, c.product_url, coalesce(c.image_url,''), coalesce(c.all_images,'')
- FROM enrichment_tracking et JOIN versa_catalog c ON c.mfr_sku=et.mfr_sku
- WHERE ${where} ORDER BY c.mfr_sku ${LIMIT ? 'LIMIT ' + LIMIT : ''}`)
- .split('\n').filter(Boolean)
- .map((l) => { const [mfr_sku, product_url, image_url, all_images] = l.split('|'); return { mfr_sku, product_url, image_url, all_images }; });
+function sqlLit(s) { return "'" + String(s).replace(/'/g, "''") + "'"; }
-console.log(`[recover-versa] ${APPLY ? 'APPLY' : 'DRY-RUN'} — ${rows.length} dead versa rows${LIMIT ? ` (limit ${LIMIT})` : ''}`);
-const recMap = join(DATA, 'versa-recovery-map.jsonl');
-const resMap = join(DATA, 'versa-restore-map.jsonl');
-writeFileSync(recMap, ''); writeFileSync(resMap, '');
+async function main() {
+ const where = `et.catalog_table='versa_catalog' AND et.phase3_ai_at IS NULL AND et.last_error='Gemini returned no data' AND coalesce(c.product_url,'')<>''`;
+ const rows = psql(
+ `SELECT c.mfr_sku, c.product_url, coalesce(c.image_url,''), coalesce(c.all_images,'')
+ FROM enrichment_tracking et JOIN versa_catalog c ON c.mfr_sku=et.mfr_sku
+ WHERE ${where} ORDER BY c.mfr_sku ${LIMIT ? 'LIMIT ' + LIMIT : ''}`)
+ .split('\n').filter(Boolean)
+ .map((l) => { const [mfr_sku, product_url, image_url, all_images] = l.split('|'); return { mfr_sku, product_url, image_url, all_images }; });
-let recovered = 0, nopdp = 0, noimg = 0, deadnew = 0;
-const pageCache = new Map();
-const CONC = 6;
-for (let i = 0; i < rows.length; i += CONC) {
- const batch = rows.slice(i, i + CONC);
- await Promise.all(batch.map(async (r) => {
- let html = pageCache.get(r.product_url);
- if (html === undefined) { html = await getText(r.product_url); pageCache.set(r.product_url, html); }
- if (!html) { nopdp++; return; }
- const url = extractImage(html, r.mfr_sku);
- if (!url) { noimg++; return; }
- if (isDataUri(url)) { noimg++; return; } // never persist a data: URI
- const live = await isLiveImage(url);
- if (!live) { deadnew++; return; }
- recovered++;
- appendFileSync(recMap, JSON.stringify({ mfr_sku: r.mfr_sku, new_image_url: url, verified: true }) + '\n');
- appendFileSync(resMap, JSON.stringify({ mfr_sku: r.mfr_sku, old_image_url: r.image_url, old_all_images: r.all_images }) + '\n');
- }));
- process.stdout.write(`\r progress ${Math.min(i + CONC, rows.length)}/${rows.length} recovered=${recovered} noimg=${noimg} nopdp=${nopdp} deadnew=${deadnew} `);
-}
-console.log('');
-const rate = rows.length ? ((recovered / rows.length) * 100).toFixed(1) : '0';
-console.log(`[recover-versa] recovered ${recovered}/${rows.length} = ${rate}% (noimg=${noimg} nopdp=${nopdp} new-url-not-live=${deadnew})`);
-console.log(` recovery map: ${recMap}`);
-console.log(` restore map: ${resMap}`);
+ console.log(`[recover-versa] ${APPLY ? 'APPLY' : 'DRY-RUN'} — ${rows.length} dead versa rows${LIMIT ? ` (limit ${LIMIT})` : ''}`);
+ const recMap = join(DATA, 'versa-recovery-map.jsonl');
+ const resMap = join(DATA, 'versa-restore-map.jsonl');
+ writeFileSync(recMap, ''); writeFileSync(resMap, '');
+
+ let recovered = 0, nopdp = 0, noimg = 0, deadnew = 0;
+ const pageCache = new Map();
+ const CONC = 6;
+ for (let i = 0; i < rows.length; i += CONC) {
+ const batch = rows.slice(i, i + CONC);
+ await Promise.all(batch.map(async (r) => {
+ let html = pageCache.get(r.product_url);
+ if (html === undefined) { html = await getText(r.product_url); pageCache.set(r.product_url, html); }
+ if (!html) { nopdp++; return; }
+ const url = extractImage(html, r.mfr_sku);
+ if (!url) { noimg++; return; }
+ if (isDataUri(url)) { noimg++; return; } // never persist a data: URI
+ const live = await isLiveImage(url);
+ if (!live) { deadnew++; return; }
+ recovered++;
+ appendFileSync(recMap, JSON.stringify({ mfr_sku: r.mfr_sku, new_image_url: url, verified: true }) + '\n');
+ appendFileSync(resMap, JSON.stringify({ mfr_sku: r.mfr_sku, old_image_url: r.image_url, old_all_images: r.all_images }) + '\n');
+ }));
+ process.stdout.write(`\r progress ${Math.min(i + CONC, rows.length)}/${rows.length} recovered=${recovered} noimg=${noimg} nopdp=${nopdp} deadnew=${deadnew} `);
+ }
+ console.log('');
+ const rate = rows.length ? ((recovered / rows.length) * 100).toFixed(1) : '0';
+ console.log(`[recover-versa] recovered ${recovered}/${rows.length} = ${rate}% (noimg=${noimg} nopdp=${nopdp} new-url-not-live=${deadnew})`);
+ console.log(` recovery map: ${recMap}`);
+ console.log(` restore map: ${resMap}`);
-if (APPLY) {
- // GATED: write fresh image_url into versa_catalog (Mac2 canonical *_catalog).
- // NOTE: phase-3 enrichment reads the KAMATERA catalog — after this Mac2 write,
- // the Mac2->Kamatera *_catalog sync must carry it over for the loop to clear.
- const lines = require('node:fs').readFileSync(recMap, 'utf8').split('\n').filter(Boolean);
- let wrote = 0;
- for (const ln of lines) {
- const { mfr_sku, new_image_url } = JSON.parse(ln);
- psql(`UPDATE versa_catalog SET image_url=${sqlLit(new_image_url)} WHERE mfr_sku=${sqlLit(mfr_sku)}`);
- wrote++;
+ if (APPLY) {
+ // GATED: write fresh image_url into versa_catalog (Mac2 canonical *_catalog).
+ // NOTE: phase-3 enrichment reads the KAMATERA catalog — after this Mac2 write,
+ // the Mac2->Kamatera *_catalog sync must carry it over for the loop to clear.
+ const lines = readFileSync(recMap, 'utf8').split('\n').filter(Boolean);
+ let wrote = 0;
+ for (const ln of lines) {
+ const { mfr_sku, new_image_url } = JSON.parse(ln);
+ psql(`UPDATE versa_catalog SET image_url=${sqlLit(new_image_url)} WHERE mfr_sku=${sqlLit(mfr_sku)}`);
+ wrote++;
+ }
+ console.log(`[recover-versa] APPLIED ${wrote} image_url updates to versa_catalog (Mac2).`);
}
- console.log(`[recover-versa] APPLIED ${wrote} image_url updates to versa_catalog (Mac2).`);
}
-function sqlLit(s) { return "'" + String(s).replace(/'/g, "''") + "'"; }
+
+// Only run the DB-driven recovery when executed directly; importing the module
+// (e.g. from tests) exposes the pure helpers without touching the database.
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ await main();
+}
diff --git a/scripts/test-recover-versa.mjs b/scripts/test-recover-versa.mjs
new file mode 100644
index 0000000..8386aa0
--- /dev/null
+++ b/scripts/test-recover-versa.mjs
@@ -0,0 +1,63 @@
+#!/usr/bin/env node
+// test-recover-versa.mjs — TK-11048 negative-case unit tests for the Versa
+// image-recovery pure helpers. Imports the REAL functions from recover-versa.mjs
+// (the direct-run guard means importing does NOT touch the database).
+//
+// The GATE is the deterministic, offline, $0 pure-function suite. Network checks
+// for isLiveImage are best-effort: a network hiccup marks SKIP, never a false FAIL.
+//
+// node scripts/test-recover-versa.mjs
+import { isDataUri, extractImage, isLiveImage } from './recover-versa.mjs';
+
+let pass = 0, fail = 0, skip = 0;
+const fails = [];
+function ok(cond, name) { if (cond) { pass++; } else { fail++; fails.push(name); console.log(` FAIL ${name}`); } }
+
+// ── isDataUri: must catch the exact original bug shape ───────────────────────
+// The versa scraper stored the base64 lazy-load placeholder with the host
+// prepended, e.g. https://host/data:image/jpeg;base64,.... All must be rejected.
+ok(isDataUri('data:image/png;base64,AAAA') === true, 'isDataUri: bare data: URI');
+ok(isDataUri('https://www.versadesignedsurfaces.com/data:image/jpeg;base64,/9j/4AAQ') === true, 'isDataUri: host-prefixed data: URI (the real bug)');
+ok(isDataUri('anything;base64,ZZZ') === true, 'isDataUri: ;base64, marker anywhere');
+ok(isDataUri('https://www.versadesignedsurfaces.com/fileadmin/_processed_/x/csm_A.jpg') === false, 'isDataUri: real fileadmin URL accepted');
+ok(isDataUri('') === false, 'isDataUri: empty string not a data URI');
+ok(isDataUri(null) === false, 'isDataUri: null not a data URI');
+
+// ── extractImage: never persists a data: URI; needs the SKU core to match ────
+const dataOnlyHtml = `<img src="data:image/jpeg;base64,/9j/4AAQSkZJRg" data-lazy="1">`;
+ok(extractImage(dataOnlyHtml, 'AVF25-834') === null, 'extractImage: page with ONLY a data: URI -> null (never persists placeholder)');
+
+const realHtml = `<picture><source srcset="/fileadmin/_processed_/d/6/csm_Terra_AVF25-834-Ice_df75.jpg 800w"></picture>`;
+const got = extractImage(realHtml, 'AVF25-834');
+ok(got === 'https://www.versadesignedsurfaces.com/fileadmin/_processed_/d/6/csm_Terra_AVF25-834-Ice_df75.jpg',
+ 'extractImage: real /fileadmin/ URL for matching SKU -> absolute URL');
+
+const wrongSkuHtml = `<img src="/fileadmin/_processed_/x/csm_Other_ZZ99-000-Foo.jpg">`;
+ok(extractImage(wrongSkuHtml, 'AVF25-834') === null, 'extractImage: no fileadmin path contains SKU core -> null');
+
+const emptyHtml = `<html><body>no images here</body></html>`;
+ok(extractImage(emptyHtml, 'AVF25-834') === null, 'extractImage: no images at all -> null');
+
+// A fileadmin path that IS a data URI can never be produced by the /fileadmin/
+// regex, but the belt-and-braces isDataUri reject inside extractImage still holds:
+ok(extractImage(dataOnlyHtml, 'ZZ99-000') === null, 'extractImage: unmatched SKU on data-only page -> null');
+
+// ── isLiveImage: network, best-effort (the "200 that is really HTML" class) ──
+async function net() {
+ try {
+ // Negative: the site homepage returns 200 text/html — must NOT count as a live image.
+ const htmlNotImage = await isLiveImage('https://www.versadesignedsurfaces.com/');
+ ok(htmlNotImage === false, 'isLiveImage: 200 text/html homepage -> false (not an image)');
+ } catch { skip++; console.log(' SKIP isLiveImage html-negative (network)'); }
+ try {
+ // Positive control: one already-recovered, liveness-verified URL from the map.
+ const known = 'https://www.versadesignedsurfaces.com/fileadmin/_processed_/d/6/csm_Terra_AVF01-025-Ice_Blink_PVC-free_df75398070.jpg';
+ const liveImg = await isLiveImage(known);
+ if (liveImg) { pass++; } else { skip++; console.log(' SKIP isLiveImage image-positive (network/url drift)'); }
+ } catch { skip++; console.log(' SKIP isLiveImage image-positive (network)'); }
+}
+await net();
+
+console.log(`\n[test-recover-versa] PASS=${pass} FAIL=${fail} SKIP=${skip}`);
+if (fail > 0) { console.log('FAILED:', fails.join('; ')); process.exit(1); }
+process.exit(0);
diff --git a/verification/e2e-proof.json b/verification/e2e-proof.json
new file mode 100644
index 0000000..80020c9
--- /dev/null
+++ b/verification/e2e-proof.json
@@ -0,0 +1,36 @@
+{
+ "ticket": "TK-11048",
+ "agent": "iterm-tk11048-recovery",
+ "a2a": { "task_id": "TK-11048", "finalizer": "/root", "child": "iterm-tk11048-recovery", "chain": "/root->iterm-tk11048-recovery", "depth": 1 },
+ "intent": "Local/read-only verification of the Versa feed-first dead-image recovery: prove deterministic recovery, independent source-URL liveness, guarded SQL + rollback, and negative-case tests — without any canonical DB, Shopify, deploy, or remote write.",
+ "risk_tier": "R0-local-readonly",
+ "environment": "Mac2 local; ~/Projects/dw-dead-image-recovery; public read-only GET to versadesignedsurfaces.com; no DB writes",
+ "timestamp": "2026-09-02T00:00:00Z",
+ "checks": [
+ {"name": "syntax recover-versa.mjs", "command": "node --check scripts/recover-versa.mjs", "verdict": "PASS"},
+ {"name": "syntax test", "command": "node --check scripts/test-recover-versa.mjs", "verdict": "PASS"},
+ {"name": "negative-case unit tests", "command": "node scripts/test-recover-versa.mjs", "verdict": "PASS", "detail": "PASS=13 FAIL=0 SKIP=0"},
+ {"name": "deterministic dry-run (artifact)", "command": "recover-versa.mjs full DRY-RUN (data/versa-fullrun.log)", "verdict": "PASS", "detail": "recovered 345/345 = 100.0% (noimg=0 nopdp=0 new-url-not-live=0)"},
+ {"name": "recovery-map integrity", "command": "grep/wc data/versa-recovery-map.jsonl", "verdict": "PASS", "detail": "345 rows, all verified:true, 0 data:/base64 URIs persisted"},
+ {"name": "restore-map reversibility", "command": "wc -l data/versa-restore-map.jsonl", "verdict": "PASS", "detail": "345 rows capturing old_image_url (base64 placeholders) — the undo record"},
+ {"name": "independent source-URL liveness", "command": "fresh GET of every 8th recovered URL (40 sample)", "verdict": "PASS", "detail": "PASS=40 FAIL=0 — 200 + Content-Type image/* weeks after original run"},
+ {"name": "negative control (old URL dead)", "command": "isLiveImage(restore-map old_image_url)", "verdict": "PASS", "detail": "isLiveImage=false on the base64 placeholder -> broken->working proven"},
+ {"name": "guarded apply SQL", "command": "structural scan data/apply-versa.sql", "verdict": "PASS", "detail": "single BEGIN..COMMIT; 345 UPDATE; every UPDATE carries WHERE mfr_sku= AND broken-image guard (data:image|base64|non-http); 0 UPDATEs missing WHERE; idempotent, cannot clobber a healthy image"},
+ {"name": "apply-path bug fix + testability", "command": "node --check + import from test", "verdict": "PASS", "detail": "fixed ESM require('node:fs') -> imported readFileSync (the gated --apply path would have thrown ReferenceError); added direct-run guard so importing exposes pure helpers without a DB query"},
+ {"name": "local DB dry-run re-run on Mac2", "command": "recover-versa.mjs --limit 60 (read-only)", "verdict": "SKIP", "reason": "Mac2 psql /tmp socket denied by sandbox (Operation not permitted) — matches prior tk11048_local_review finding; not required, artifacts + independent network liveness stand as proof"}
+ ],
+ "assertions": [
+ "817 dead image_url rows are a MIX of root causes -> per-vendor triage (established on ticket)",
+ "Versa root cause: scraper persisted a host-prefixed base64 data-URI placeholder in image_url; the real image lives in the live product-page /fileadmin/ HTML",
+ "Versa: 345/345 dead rows recoverable $0 feed-first, every new URL liveness-verified (200 + image/*)",
+ "0 data:/base64 URIs leaked into the recovery map (hard-rejected at extract and persist)",
+ "restore map preserves every original value -> the apply is fully reversible",
+ "apply-versa.sql only fires on still-broken rows and is idempotent",
+ "thibaut cohort is 0% recoverable (discontinued; dead CloudFront CDN; full-feed proven complete) -> stamp, do not re-scrape",
+ "zero executable mutations performed by this session"
+ ],
+ "side_effects": {"shopify_writes": 0, "database_writes": 0, "activation": 0, "deploy": 0, "sends": 0, "spend": 0, "dns": 0, "remote_push": 0, "deletes": 0, "schedule_changes": 0},
+ "cleanup": "No external state created; only local repo files (test + this proof) added and recover-versa.mjs edited in place.",
+ "verdict": "PASS",
+ "blocker": "Two canonical writes remain Steve-gated and were NOT executed: (A) versa image_url backfill — must land on the KAMATERA versa_catalog (canonical for prod-serving; Mac2=1745 vs Kamatera=2423 rows differ) via apply-versa.sql; (B) thibaut discontinue stamp. This session's scope is local/read-only verification only."
+}
← 94c41e5 allow Carlisle staging while keeping it offline
·
back to Dw Dead Image Recovery
·
TK-11048: Kamatera-scoped Versa re-recovery — 145/283 recove bd89bdd →