[object Object]

← back to Gmc Titlefix

TK-11233: verify must not report a vacuous pass on an empty run-log

a07ace521f6b0d57e6511120d7cabc93b9bdaad9 · 2026-09-10 08:12:48 -0700 · Steve Abrams

verify exited 0 with '0/0 PASS' when a run-log contained no successful
applies - a false green of exactly the kind the 2026-09-05 addendum warns
about (the script's own success line is not proof). If everything drifted or
failed, the canary would have looked like it passed.

Now exits 6 as INCONCLUSIVE, prints the not-applied reason breakdown, and
states plainly that nothing was verified. Absence of evidence is reported as
a failure to verify, never as success.

74/74 tests pass (was 66; +8 covering verify/rollback arg handling and the
two vacuous-pass paths).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01W43bYwipr6GHKjJtWSsMyX

Files touched

Diff

commit a07ace521f6b0d57e6511120d7cabc93b9bdaad9
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Thu Sep 10 08:12:48 2026 -0700

    TK-11233: verify must not report a vacuous pass on an empty run-log
    
    verify exited 0 with '0/0 PASS' when a run-log contained no successful
    applies - a false green of exactly the kind the 2026-09-05 addendum warns
    about (the script's own success line is not proof). If everything drifted or
    failed, the canary would have looked like it passed.
    
    Now exits 6 as INCONCLUSIVE, prints the not-applied reason breakdown, and
    states plainly that nothing was verified. Absence of evidence is reported as
    a failure to verify, never as success.
    
    74/74 tests pass (was 66; +8 covering verify/rollback arg handling and the
    two vacuous-pass paths).
    
    Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
    Claude-Session: https://claude.ai/code/session_01W43bYwipr6GHKjJtWSsMyX
---
 test/tk11233-v2-guards.test.mjs | 20 ++++++++++++++++++++
 tk11233-umbrella-hires-v2.mjs   | 19 ++++++++++++++++++-
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/test/tk11233-v2-guards.test.mjs b/test/tk11233-v2-guards.test.mjs
index 7ecd8ef..f378ba2 100644
--- a/test/tk11233-v2-guards.test.mjs
+++ b/test/tk11233-v2-guards.test.mjs
@@ -149,6 +149,26 @@ for (const [name, env] of [['no env', {}], ['stale/other approval value', { TK11
 // unknown subcommand
 ok('rejects unknown subcommand', runCli(['frobnicate']).code === 4);
 
+// ---------------------------------------------------------------- 5. verify cannot report a false green
+console.log('\n[5] verify (D4) - absence of evidence is never reported as success');
+const emptyLog = path.join(tmp, 'empty.jsonl'); fs.writeFileSync(emptyLog, '');
+const failLog = path.join(tmp, 'fail.jsonl');
+fs.writeFileSync(failLog, JSON.stringify({ ok: false, why: 'live-drift-since-select' }) + '\n' + JSON.stringify({ ok: false, status: 'FAILED' }) + '\n');
+{
+  const r = runCli(['verify', '--runlog', emptyLog]);
+  ok('verify on an EMPTY run-log is INCONCLUSIVE, not "0/0 PASS" (exit ' + r.code + ')', r.code === 6, r.out.trim().split('\n').slice(-1)[0]);
+  ok('  ...and says nothing was verified', /INCONCLUSIVE/.test(r.out) && /Do NOT treat this as a passing canary/.test(r.out));
+}
+{
+  const r = runCli(['verify', '--runlog', failLog]);
+  ok('verify on a run-log with only FAILURES is INCONCLUSIVE (exit ' + r.code + ')', r.code === 6);
+  ok('  ...and reports the not-applied reasons', /live-drift-since-select/.test(r.out) && /FAILED/.test(r.out));
+}
+for (const cmd of ['verify', 'rollback']) {
+  ok(cmd + ' refuses without --runlog', runCli([cmd]).code === 4);
+  ok(cmd + ' refuses a missing run-log file', runCli([cmd, '--runlog', path.join(tmp, 'nope.jsonl')]).code === 4);
+}
+
 fs.rmSync(tmp, { recursive: true, force: true });
 console.log('\n' + pass + ' passed, ' + fail + ' failed');
 process.exit(fail ? 1 : 0);
diff --git a/tk11233-umbrella-hires-v2.mjs b/tk11233-umbrella-hires-v2.mjs
index e49130c..86d71d9 100644
--- a/tk11233-umbrella-hires-v2.mjs
+++ b/tk11233-umbrella-hires-v2.mjs
@@ -341,8 +341,25 @@ async function apply() {
 async function verify() {
   const lp = val('--runlog', null);
   if (!lp || !fs.existsSync(lp)) { console.error('--runlog <file> required'); process.exit(4); }
-  const applied = fs.readFileSync(lp, 'utf8').trim().split('\n').filter(Boolean).map(l => JSON.parse(l)).filter(r => r.ok);
+  const all = fs.readFileSync(lp, 'utf8').trim().split('\n').filter(Boolean).map(l => JSON.parse(l));
+  const applied = all.filter(r => r.ok);
+  const notApplied = all.filter(r => !r.ok);
+  // A run-log with nothing successfully applied must NOT report a vacuous "0/0 PASS" - that is a
+  // false green of exactly the kind the 2026-09-05 addendum warns about (the script's own success
+  // line is not proof). Absence of evidence is reported as a failure to verify, not as success.
+  if (!applied.length) {
+    console.error('VERIFY INCONCLUSIVE: run-log ' + lp + ' contains ' + all.length + ' entries but 0 successful applies.');
+    if (notApplied.length) {
+      console.error('  not-applied reasons:');
+      const why = {};
+      for (const r of notApplied) { const k = r.why || r.status || (r.errs && JSON.stringify(r.errs).slice(0, 60)) || r.err || 'unknown'; why[k] = (why[k] || 0) + 1; }
+      for (const k of Object.keys(why)) console.error('    ' + why[k] + '  ' + k);
+    }
+    console.error('  Nothing was verified. Do NOT treat this as a passing canary.');
+    process.exit(6);
+  }
   console.log('=== VERIFY ' + applied.length + ' applied products (independent live re-read) ===');
+  if (notApplied.length) console.log('  (' + notApplied.length + ' entries in this run-log were not applied and are not verifiable)');
   const results = [];
   for (const r of applied) {
     const chk = { shopify_id: r.shopify_id, product_mfr_sku: r.product_mfr_sku, checks: {} };

← 7877b0d TK-11233: authoritative landing_page_error classification of  ·  back to Gmc Titlefix  ·  TK-11233: derive an exact <=25-product canary cohort with it af743e5 →