← back to Dw Photo Capture
.claude/worktrees/agent-a841f65dcd871d6ac
339 lines
commit 515ac4aa08fefb38a813e87771c1c189e07c5522
Author: Steve <steve@designerwallcoverings.com>
Date: Thu Sep 24 23:43:15 2026 -0700
TK-12205: PiP worker verify on the SAME frame, recoverable fallback, ?debug=1 readout (Cody final gate)
(a) The one-shot verify compared the worker's bake of frame T against a main bake of the live <video>
read after the round-trip (frame T+100-270ms): a live-camera race (AE/AWB convergence). Reference
is now a main-thread bake of frame.clone() taken BEFORE transfer -> identical input; tolerance
tightened 40 -> 4 mean-luma (measured delta: 0). Also fixed a latent bug the old tolerance hid:
the reference canvas was never sized (default 300x150), so its histogram read out-of-bounds
zeros -> a systematic -21 luma offset.
(b) Failures (verify mismatch, worker error/{err}, >2s stall, post failure) fall back for now,
console.warn the reason, and re-arm (fresh worker + re-verify) on the next camera open via
baker.rearm(), max 3 per page load. Replies from a superseded worker are discarded (gen counter).
(c) baker.mode() / fallbackReason() / stats() / label(); ?debug=1 shows 'Color check · worker' or
'· main (reason)' in the PiP label (index) / #pipDbg (cam).
(d) Tests T9 (two-shot worker through front+back shutter->review->keep), T10 (injected mismatch ->
fallback -> re-arm on next open), T11 (cam.html tuned baker: worker, real pixels, debug readout).
37/37, 0 console errors. Tuned @4x unchanged: 0 long tasks, script 0.5%/0.4%.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
diff --git a/public/cam.html b/public/cam.html
index 6af2989..6f828bd 100644
--- a/public/cam.html
+++ b/public/cam.html
@@ -86,6 +86,8 @@
transition:width .18s,max-width .18s;
max-height:calc((100vh - 220px) / var(--ui-scale))} /* Cycle-2 fix #7: stay clear of the bottom foot band on a short (iPhone SE) viewport */
#preview[hidden]{display:none}
+ #pipDbg{position:fixed;right:12px;top:52px;z-index:9;font:700 10px/1 ui-monospace,Menlo,monospace;color:var(--gold);background:#16140fcc;border-radius:6px;padding:3px 6px} /* ?debug=1 only */
+ #pipDbg[hidden]{display:none}
/* Cycle-2 fix #7: pointer-events:none while enlarged — it auto-reverts on its own 2.5s timer, so
it never needs a second tap while big, and can't swallow taps meant for the WB-tap video/gate underneath. */
#preview.big{width:calc(min(60vw,320px,(100svh - 260px) * .75) / var(--ui-scale));max-width:calc(320px / var(--ui-scale));pointer-events:none}
@@ -110,7 +112,7 @@
</style>
</head>
<body>
- <div id="stage"><video id="v" playsinline autoplay muted></video><canvas id="preview"></canvas></div>
+ <div id="stage"><video id="v" playsinline autoplay muted></video><canvas id="preview"></canvas><span id="pipDbg" hidden></span></div>
<div id="flash"></div>
<div class="bar">
@@ -283,6 +285,7 @@ $('#preview').addEventListener('click',()=>{
let _pvLastTs=0, _histLastTs=0;
const PV_INTERVAL_MS=250, HIST_INTERVAL_MS=500;
const _pipBaker = CapturePipeline.createPipBaker(); // TK-12205 perf: off-main-thread PiP bake
+const PIP_DEBUG = /[?&]debug=1(&|$)/.test(location.search); // ?debug=1 shows which bake path runs
function previewLoop(ts){
_previewRAF=requestAnimationFrame(previewLoop);
if(!camLive) return;
@@ -294,6 +297,7 @@ function previewTick(ts){
const v=$('#v'), pc=$('#preview');
const neutral = CapturePipeline.isNeutral(_tune, _wbGains);
pc.hidden = neutral;
+ const dbg = $('#pipDbg'); dbg.hidden = !PIP_DEBUG || neutral;
if(neutral) return;
const pw = Math.min(240, Math.max(v.videoWidth, v.videoHeight));
const scale = pw / Math.max(v.videoWidth, v.videoHeight);
@@ -309,6 +313,7 @@ function previewTick(ts){
CapturePipeline.drawHistogram($('#adjHist').getContext('2d'), hist);
$('#adjClip').hidden = !(hist.clipHigh > 0.02 || hist.clipLow > 0.02);
});
+ if(PIP_DEBUG) dbg.textContent = 'Color check · ' + _pipBaker.label(); // ?debug=1 — which bake path runs on this device
}
// ── getUserMedia: rear camera, hardened for iOS Safari ──
@@ -357,6 +362,7 @@ async function startCamera(){
// awaited a bounded play() above, so by the time acquire() resolves the video is confirmed playing.
// TK-12205: rAF-gated (~4fps), not setInterval(fn,80) — see previewLoop()/previewTick() above.
if(_previewRAF) cancelAnimationFrame(_previewRAF);
+ _pipBaker.rearm(); // TK-12205: re-arm the PiP worker after a runtime fallback (fresh worker + re-verify, capped)
_pvLastTs=0; _histLastTs=0; _previewRAF=requestAnimationFrame(previewLoop);
const caps = CapturePipeline.Hardware.probe(videoTrack);
if(Object.keys(caps).length>0){
diff --git a/public/index.html b/public/index.html
index b1b4bd4..8dd6573 100644
--- a/public/index.html
+++ b/public/index.html
@@ -2325,6 +2325,7 @@ function wbSampleAt(clientX,clientY){ const v=$('#wbVideo'); if(!v.videoWidth)re
let _tsStream=null,_tsTrack=null,_tsPhase='front',_tsBusy=false,_tsLive=false,_tsFrontImg=null,_tsPvRAF=0,_tsPvLastTs=0,_tsHistLastTs=0,_tsWake=null,_tsOpening=false,_tsAcquiring=false;
const TS_PV_INTERVAL_MS=250; // ~4fps colour-check PiP bake
const TS_HIST_INTERVAL_MS=500;
+const PIP_DEBUG=/[?&]debug=1(&|$)/.test(location.search);
const _tsPipBaker=CapturePipeline.createPipBaker(); // TK-12205 perf: off-main-thread PiP bake (worker + main fallback) // ~2fps histogram, computed off the same PiP buffer
let _tsStreamGen=0; // bumped every startTsStream() call — lets a late/orphaned getUserMedia resolution (after a timeout bail-out) be detected and released instead of silently adopted
// TK-12124: getUserMedia() can hang forever on WebKit/Safari (denied-but-not-rejected permission, backgrounded
@@ -2372,6 +2373,7 @@ function tsPreviewTick(ts){
const wantHist=!ts || ts-_tsHistLastTs>=TS_HIST_INTERVAL_MS;
if(wantHist) _tsHistLastTs=ts||performance.now();
try{ _tsPipBaker.tick(v,pv,pw,ph,_tsTune,_tsWB||null,wantHist,tsShowHistogram); }catch(e){}
+ if(PIP_DEBUG && lbl) lbl.textContent='Color check · '+_tsPipBaker.label(); // ?debug=1 — which bake path runs on this device
}
// tap the PiP to enlarge it briefly (auto-reverts) — see .ts-pip .big in CSS
$('#tsPreview').addEventListener('click',()=>{
@@ -2637,6 +2639,7 @@ async function openTwoShotCam(startPhase){
tsSetStep();
$('#twoShotCam').hidden=false; document.body.style.overflow='hidden';
if(_tsLive) $('#tsShutterBtn').disabled=false; // only arm the shutter if a live frame is confirmed (startTsStream owns the enable; never re-arm a dead shutter)
+ _tsPipBaker.rearm(); // TK-12205: a runtime worker failure earlier this page load gets a fresh worker + re-verify on each camera open (capped)
if(_tsPvRAF) cancelAnimationFrame(_tsPvRAF); _tsPvLastTs=0; _tsHistLastTs=0; _tsPvRAF=requestAnimationFrame(tsPreviewLoop);
drawTsGhost();
tsRequestWake();
diff --git a/public/js/capture-pipeline.js b/public/js/capture-pipeline.js
index 1794926..322f9d3 100644
--- a/public/js/capture-pipeline.js
+++ b/public/js/capture-pipeline.js
@@ -315,33 +315,60 @@
// (a ~0.1ms ref, transferable) to a dedicated Worker that runs the IDENTICAL drawSource() + apply()
// (+ histogram()) from THIS file on an OffscreenCanvas 2D and posts back an ImageBitmap. Same math,
// same code — WYSIWYG holds. No CSS filter, no WebGL. The full-res capture path is NOT touched.
- // Feature-detected with a synchronous main-thread fallback (the previous behaviour) whenever
- // Worker/OffscreenCanvas/VideoFrame are missing, the worker errors, stalls >2s, or its FIRST result
- // disagrees with a one-off main-thread bake of the same tune (guards a silently-black worker bitmap).
+ //
+ // Safety net (Cody final gate): the FIRST worker result per worker instance is verified against a
+ // main-thread bake of the SAME frame (frame.clone() taken before transfer — identical input pixels,
+ // so a tight tolerance; a live-camera AE/AWB change between two different frames can't trip it).
+ // Any failure (verify mismatch, worker error/{err}, >2s stall, post failure) falls back to the
+ // synchronous main-thread bake FOR NOW, console.warns the reason, and re-arms (fresh worker +
+ // re-verify) on the next camera open via rearm() — capped at MAX_REARMS per page load.
+ // Observability: mode(), fallbackReason(), stats(); pages show it in the PiP label with ?debug=1.
+ const PIP_VERIFY_TOL = 4; // mean-luma units; same input frame + same code -> expect ~0
+ const PIP_MAX_REARMS = 3;
function createPipBaker(opts) {
opts = opts || {};
- let worker = null, mode = 'main', busy = false, busySince = 0, verified = false;
- const api = { mode: function () { return mode; }, fallbackReason: null, tick: tick };
- const canWorker = !opts.forceMain && typeof Worker !== 'undefined' && typeof OffscreenCanvas !== 'undefined' &&
+ let worker = null, mode = 'main', busy = false, busySince = 0, verified = false, reason = null, rearms = 0, gen = 0;
+ const stats = { workerResults: 0, mainBakes: 0, fallbacks: 0, rearms: 0, lastVerifyDelta: null };
+ const supported = !opts.forceMain && typeof Worker !== 'undefined' && typeof OffscreenCanvas !== 'undefined' &&
typeof VideoFrame !== 'undefined' && typeof ImageBitmap !== 'undefined';
- if (canWorker) {
+ const api = {
+ mode: function () { return mode; },
+ fallbackReason: function () { return reason; },
+ stats: function () { return Object.assign({ mode: mode, reason: reason, verified: verified }, stats); },
+ label: function () { return mode === 'worker' ? 'worker' + (verified ? '' : ' (verifying)') : 'main' + (reason ? ' (' + reason + ')' : ''); },
+ tick: tick, rearm: rearm,
+ _testFault: null // TEST SEAM ONLY: 'mismatch' corrupts the NEXT verify. Nothing in app code sets it.
+ };
+ function start() {
+ if (!supported) { mode = 'main'; reason = 'unsupported'; return; }
try {
- worker = new Worker(opts.workerUrl || '/js/pip-worker.js');
- worker.onerror = function () { fallback('worker-error'); };
- mode = 'worker';
- } catch (e) { worker = null; api.fallbackReason = 'worker-ctor:' + (e && e.name); }
- } else api.fallbackReason = 'unsupported';
+ const w = new Worker(opts.workerUrl || '/js/pip-worker.js'), myGen = ++gen;
+ w.onerror = function (e) { if (myGen === gen) fallback('worker-error' + (e && e.message ? ':' + String(e.message).slice(0, 60) : '')); };
+ worker = w; mode = 'worker'; verified = false; busy = false; reason = null;
+ } catch (e) { worker = null; mode = 'main'; reason = 'worker-ctor:' + (e && e.name); }
+ }
function fallback(why) {
- mode = 'main'; busy = false; api.fallbackReason = why;
if (worker) { try { worker.terminate(); } catch (e) {} worker = null; }
+ gen++; // orphan any in-flight reply from the dead worker
+ mode = 'main'; busy = false; reason = why; stats.fallbacks++;
+ try { console.warn('[CapturePipeline] PiP worker -> main-thread fallback:', why); } catch (e) {}
+ }
+ // Called on every camera open: if a RUNTIME failure dropped us to main, try the worker again.
+ function rearm() {
+ if (mode === 'worker' || !supported) return false;
+ if (rearms >= PIP_MAX_REARMS) return false;
+ rearms++; stats.rearms++;
+ start();
+ return mode === 'worker';
}
- function mainBake(video, canvas, w, h, tune, preGain, wantHist) {
+ function mainBake(src, canvas, w, h, tune, preGain, wantHist) {
const ctx = canvas.getContext('2d', { willReadFrequently: true });
- drawSource(ctx, video, w, h, tune.straighten);
+ drawSource(ctx, src, w, h, tune.straighten);
apply(ctx, w, h, tune, { capture: false, preGain: preGain || null });
return wantHist ? histogram(ctx, w, h) : null;
}
function meanLuma(hist) {
+ if (!hist || !hist.lum) return -1;
let n = 0, s = 0;
for (let i = 0; i < 256; i++) { n += hist.lum[i]; s += i * hist.lum[i]; }
return n ? s / n : -1;
@@ -356,33 +383,55 @@
let frame = null;
try { frame = new VideoFrame(video); } catch (e) { frame = null; } // no decoded frame yet -> main path this tick
if (frame) {
- busy = true; busySince = performance.now();
+ let refMean = null;
const needVerify = !verified;
- worker.onmessage = function (e) {
- const m = e.data || {};
- busy = false;
- if (m.err) { if (m.bm) m.bm.close(); fallback('worker:' + m.err); return; }
- if (needVerify) {
- verified = true;
- const ref = mainBake(video, document.createElement('canvas'), w, h, tune, preGain, true);
- const a = meanLuma(m.hist), b = meanLuma(ref);
- if (a < 0 || Math.abs(a - b) > 40) { m.bm.close(); fallback('verify-mismatch:' + Math.round(a) + '/' + Math.round(b)); return; }
- }
- const ctx = canvas.getContext('2d', { willReadFrequently: true });
- ctx.clearRect(0, 0, canvas.width, canvas.height);
- ctx.drawImage(m.bm, 0, 0);
- m.bm.close();
- if (m.hist && onHist && wantHist) onHist(m.hist);
- };
- try {
- worker.postMessage({ frame: frame, w: w, h: h, tune: tune, preGain: preGain || null, wantHist: !!(wantHist || needVerify) }, [frame]);
- return;
- } catch (e) { busy = false; try { frame.close(); } catch (e2) {} fallback('post:' + (e && e.name)); }
+ if (needVerify) {
+ // reference = main-thread bake of the SAME frame (clone shares the pixels), taken BEFORE transfer
+ let ref = null;
+ try {
+ ref = frame.clone();
+ const rc = document.createElement('canvas'); rc.width = w; rc.height = h; // MUST size it: a default 300x150 canvas made the reference read out-of-bounds zeros (-21 luma) — the old ±40 tolerance hid that
+ refMean = meanLuma(mainBake(ref, rc, w, h, tune, preGain, true));
+ } catch (e) { refMean = null; }
+ try { ref && ref.close(); } catch (e) {}
+ if (refMean == null) { try { frame.close(); } catch (e) {} fallback('verify-ref-failed'); frame = null; }
+ }
+ if (frame) {
+ busy = true; busySince = performance.now();
+ const myGen = gen, fault = needVerify ? api._testFault : null;
+ if (needVerify) api._testFault = null;
+ worker.onmessage = function (e) {
+ if (myGen !== gen) { const d = e.data || {}; if (d.bm) d.bm.close(); return; } // reply from a superseded worker
+ const m = e.data || {};
+ busy = false;
+ if (m.err) { if (m.bm) m.bm.close(); fallback('worker:' + m.err); return; }
+ if (needVerify) {
+ let a = meanLuma(m.hist);
+ if (fault === 'mismatch') a = a + 100;
+ const delta = a < 0 ? Infinity : Math.abs(a - refMean);
+ stats.lastVerifyDelta = isFinite(delta) ? Math.round(delta * 100) / 100 : null;
+ if (!(delta <= PIP_VERIFY_TOL)) { m.bm.close(); fallback('verify-mismatch:' + Math.round(a) + '/' + Math.round(refMean)); return; }
+ verified = true;
+ }
+ stats.workerResults++;
+ const ctx = canvas.getContext('2d', { willReadFrequently: true });
+ ctx.clearRect(0, 0, canvas.width, canvas.height);
+ ctx.drawImage(m.bm, 0, 0);
+ m.bm.close();
+ if (m.hist && onHist && wantHist) onHist(m.hist);
+ };
+ try {
+ worker.postMessage({ frame: frame, w: w, h: h, tune: tune, preGain: preGain || null, wantHist: !!(wantHist || needVerify) }, [frame]);
+ return;
+ } catch (e) { busy = false; try { frame.close(); } catch (e2) {} fallback('post:' + (e && e.name)); }
+ }
}
}
+ stats.mainBakes++;
const hist = mainBake(video, canvas, w, h, tune, preGain, wantHist);
if (hist && onHist) onHist(hist);
}
+ start();
return api;
}
diff --git a/verification/tk12205/scripts/test-cycle2.js b/verification/tk12205/scripts/test-cycle2.js
index 3ee0888..ed0cb3a 100644
--- a/verification/tk12205/scripts/test-cycle2.js
+++ b/verification/tk12205/scripts/test-cycle2.js
@@ -273,6 +273,71 @@ function check(name, cond, detail) {
await cp.close();
}
+ // ═══════════════════ TEST 9-11: PiP worker path (Cody final gate) — tuned, fresh context ═══════════════════
+ const TUNED = '{"exposure":20,"temp":15,"contrast":10,"saturation":10}';
+ const ctx2 = await browser.newContext({ httpCredentials: { username: 'admin', password: 'DW2024!' }, viewport: { width: 390, height: 844 }, permissions: ['camera'] });
+ await ctx2.addInitScript(`try{localStorage.setItem('dwTsTune','${TUNED}');localStorage.setItem('dwCamTune','${TUNED}')}catch(e){}`);
+ const wireErrors = (pg, tag) => { pg.on('console', m => { if (m.type() === 'error') consoleErrors.push(tag + ': ' + m.text()); }); pg.on('pageerror', e => consoleErrors.push(tag + ' pageerror: ' + e.message)); };
+
+ // T9: two-shot reaches worker mode with a tuned PiP and STAYS there through shutter -> review -> keep (front AND back).
+ const p9 = await ctx2.newPage(); wireErrors(p9, 'T9');
+ await p9.goto('http://localhost:9931/?debug=1', { waitUntil: 'networkidle' });
+ await p9.click('text=Add New SKU');
+ await p9.waitForTimeout(1800);
+ let st = await p9.evaluate(() => _tsPipBaker.stats());
+ const lbl9 = await p9.evaluate(() => document.getElementById('tsPipLbl').textContent);
+ check('T9a: two-shot PiP runs in the WORKER (verified, results flowing) with a tuned PiP', st.mode === 'worker' && st.verified && st.workerResults > 0 && st.reason === null, st);
+ check('T9b: ?debug=1 PiP label names the bake path', /worker/.test(lbl9), { lbl9 });
+ await p9.click('#tsShutterBtn');
+ await p9.waitForFunction(() => !document.getElementById('tsReview').hidden, null, { timeout: 10000 });
+ await p9.click('#trKeepBtn');
+ await p9.waitForTimeout(1500);
+ const mid = await p9.evaluate(() => _tsPipBaker.stats());
+ check('T9c: still worker after front shutter->review->keep, and still baking in the BACK phase', mid.mode === 'worker' && mid.workerResults > st.workerResults && mid.fallbacks === 0, { before: st, after: mid });
+ await p9.click('#tsShutterBtn');
+ await p9.waitForFunction(() => !document.getElementById('tsReview').hidden, null, { timeout: 10000 });
+ await p9.click('#trKeepBtn');
+ await p9.waitForTimeout(1200);
+ const end9 = await p9.evaluate(() => _tsPipBaker.stats());
+ check('T9d: full two-photo flow finished with the baker still in worker mode, 0 fallbacks', end9.mode === 'worker' && end9.fallbacks === 0, end9);
+ await p9.close();
+
+ // T10: forced verify mismatch -> falls back to main (reason recorded) -> re-arms on the NEXT camera open.
+ const p10 = await ctx2.newPage(); wireErrors(p10, 'T10');
+ await p10.goto('http://localhost:9931/?debug=1', { waitUntil: 'networkidle' });
+ await p10.evaluate(() => { _tsPipBaker._testFault = 'mismatch'; });
+ await p10.click('text=Add New SKU');
+ await p10.waitForTimeout(1800);
+ const f1 = await p10.evaluate(() => ({ s: _tsPipBaker.stats(), lbl: document.getElementById('tsPipLbl').textContent }));
+ check('T10a: injected bad worker result -> main-thread fallback with a recorded reason', f1.s.mode === 'main' && /^verify-mismatch/.test(f1.s.reason || '') && f1.s.fallbacks === 1 && /main \(verify-mismatch/.test(f1.lbl), f1);
+ await p10.click('#tsClose');
+ await p10.waitForTimeout(400);
+ await p10.evaluate(() => openTwoShotCam('front'));
+ await p10.waitForTimeout(1800);
+ const f2 = await p10.evaluate(() => _tsPipBaker.stats());
+ check('T10b: next camera open re-arms a fresh worker, re-verifies, and runs in worker mode again', f2.mode === 'worker' && f2.verified && f2.rearms === 1 && f2.reason === null && f2.workerResults > 0, f2);
+ await p10.close();
+
+ // T11: cam.html with a tuned PiP actually exercises ITS baker (worker, verified, real pixels, debug readout).
+ const p11 = await ctx2.newPage(); wireErrors(p11, 'T11');
+ await p11.goto('http://localhost:9931/cam?debug=1', { waitUntil: 'load' });
+ await p11.click('#gBtn');
+ await p11.waitForFunction(() => typeof camLive !== 'undefined' && camLive, null, { timeout: 15000 });
+ await p11.waitForTimeout(1800);
+ const c1 = await p11.evaluate(() => {
+ const pc = document.getElementById('preview'), d = pc.getContext('2d', { willReadFrequently: true }).getImageData(0, 0, pc.width, pc.height).data;
+ let a = 0, rgb = 0; for (let i = 0; i < d.length; i += 4) { a += d[i + 3]; rgb += d[i] + d[i + 1] + d[i + 2]; }
+ return { s: _pipBaker.stats(), hidden: pc.hidden, dims: [pc.width, pc.height], meanA: a / (d.length / 4), meanRGB: rgb / (d.length / 4) / 3, dbg: document.getElementById('pipDbg').textContent, dbgHidden: document.getElementById('pipDbg').hidden };
+ });
+ check('T11a: cam.html tuned PiP runs in the WORKER (verified, results flowing)', c1.s.mode === 'worker' && c1.s.verified && c1.s.workerResults > 0, c1.s);
+ check('T11b: cam.html PiP shows real baked pixels (visible, opaque, not black)', !c1.hidden && c1.meanA > 250 && c1.meanRGB > 10, { hidden: c1.hidden, dims: c1.dims, meanA: c1.meanA, meanRGB: c1.meanRGB });
+ check('T11c: cam.html ?debug=1 readout names the bake path', !c1.dbgHidden && /worker/.test(c1.dbg), { dbg: c1.dbg, dbgHidden: c1.dbgHidden });
+ await p11.waitForTimeout(1500);
+ const c2 = await p11.evaluate(() => _pipBaker.stats());
+ check('T11d: cam.html stays in worker mode over time (0 fallbacks)', c2.mode === 'worker' && c2.fallbacks === 0 && c2.workerResults > c1.s.workerResults, c2);
+ await p11.close();
+ await ctx2.close();
+
console.log('\nCONSOLE ERRORS:', consoleErrors.length);
consoleErrors.forEach(e => console.log(' ERR:', e));
console.log(`\nRESULT: ${PASS} passed, ${FAIL} failed`);