`verify.mjs`

205 lines
#!/usr/bin/env node
// approval-verifier — did an approved yolo-queue item actually LAND end-to-end?
//
// The yolo-queue tracks that a task was *processed* (done/ vs failed/), but NOT
// that its real-world EFFECT landed (a memo can be prose-only, self-gate, or
// land "cancelled"). This tool reads memo .md files, derives machine-checkable
// assertions, runs them, and reports per item:
//
//   LANDED        explicit VERIFY checks all passed                 (high conf)
//   FAILED        explicit VERIFY checks present, some failed       (high conf)
//   LIKELY-LANDED auto-observed resource state matches the memo's action verb
//   LIKELY-NOOP   auto-observed state contradicts the action verb   (didn't take)
//   OBSERVED      resources found, direction ambiguous — evidence printed
//   UNVERIFIABLE  no machine-checkable signal — surfaced, never silently "done"
//
// Verification signal priority:
//   1. Explicit  <!-- VERIFY: <shell test> -->  lines, or a ```verify fenced block.
//      Each line is a shell command; exit 0 = that check passed. AUTHORITATIVE.
//   2. Auto-inference (best-effort, no false "done"): launchd labels (com.steve.*),
//      pm2 process names, and explicitly-claimed file paths — checked against live
//      state, with the memo's nearby action verb used to infer expected direction.
//   3. none → UNVERIFIABLE.
//
// Usage:
//   node verify.mjs [--dir <path>] [--memo <file>] [--json] [--status LANDED,FAILED]
//   default --dir = ~/.claude/yolo-queue/pending-approval/done   (approved+processed)
//   --pending  scan ~/.claude/yolo-queue/pending-approval (the live drafts)
//   --done     also scan ~/.claude/yolo-queue/done
// Writes a JSONL ledger to ~/.claude/yolo-queue/.approval-verify-ledger.jsonl

import { readFileSync, writeFileSync, readdirSync, existsSync, appendFileSync, statSync } from "node:fs";
import { execSync } from "node:child_process";
import { homedir } from "node:os";
import { join, basename } from "node:path";

const HOME = homedir();
const QUEUE = join(HOME, ".claude/yolo-queue");
const LEDGER = join(QUEUE, ".approval-verify-ledger.jsonl");
const args = process.argv.slice(2);
const flag = (n) => args.includes(n);
const opt = (n, d) => { const i = args.indexOf(n); return i >= 0 && args[i + 1] ? args[i + 1] : d; };

const sh = (cmd, timeout = 5000) => {
  try { return { ok: true, out: execSync(cmd, { timeout, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim() }; }
  catch (e) { return { ok: false, out: (e.stdout || "").toString().trim() }; }
};

// --- live state gathered once ---
const launchdLoaded = (() => {
  const r = sh("launchctl list", 4000);
  const set = new Set();
  if (r.ok) for (const line of r.out.split("\n")) { const c = line.split(/\s+/); if (c[2]) set.add(c[2]); }
  return set; // labels currently loaded
})();
const pm2 = (() => {
  const r = sh("pm2 jlist 2>/dev/null", 6000);
  const map = new Map();
  if (r.ok && r.out) { try { for (const p of JSON.parse(r.out)) map.set(p.name, p.pm2_env?.status || "?"); } catch {} }
  return map; // name -> status (online/stopped/...)
})();

// --- per-memo signal extraction ---
const ACTION_UP = /\b(install|installed|load|loaded|activate|activated|start|started|restart|enable|enabled|create|created|wrote|written|add|added|publish|published)\b/i;
const ACTION_DOWN = /\b(stop|stopped|bootout|boot out|retire|retired|remove|removed|delete|deleted|disable|disabled|archive|archived|decommission|kill|killed|unload|unloaded)\b/i;

function nearbyVerb(text, idx) {
  const win = text.slice(Math.max(0, idx - 60), idx + 20);
  if (ACTION_DOWN.test(win)) return "down";
  if (ACTION_UP.test(win)) return "up";
  return null;
}

function extractExplicit(text) {
  const checks = [];
  for (const m of text.matchAll(/<!--\s*VERIFY:\s*([^]*?)-->/g)) checks.push(m[1].trim());
  for (const m of text.matchAll(/```verify\s*([^]*?)```/g))
    for (const line of m[1].split("\n").map(s => s.trim()).filter(Boolean)) checks.push(line);
  return checks.filter(Boolean);
}

function extractResources(text) {
  const res = [];
  for (const m of text.matchAll(/\bcom\.steve\.[a-z0-9._-]+/gi)) {
    const label = m[0].replace(/\.plist$/i, ""); const dir = nearbyVerb(text, m.index);
    res.push({ kind: "launchd", id: label, dir, observed: launchdLoaded.has(label) ? "loaded" : "not-loaded" });
  }
  for (const [name, status] of pm2) {
    const re = new RegExp(`\\b${name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`);
    const idx = text.search(re);
    if (idx >= 0) res.push({ kind: "pm2", id: name, dir: nearbyVerb(text, idx), observed: status });
  }
  // explicitly-claimed created/written file paths only (avoid every referenced path)
  for (const m of text.matchAll(/\b(?:wrote|created|written|saved|added)\b[^\n]*?[`"]?((?:~|\/Users\/stevestudio2)\/[^\s`"')]+)/gi)) {
    let p = m[1].replace(/^~/, HOME);
    res.push({ kind: "file", id: m[1], dir: "up", observed: existsSync(p) ? "exists" : "absent" });
  }
  // dedup by kind+id
  const seen = new Set();
  return res.filter(r => { const k = r.kind + r.id; if (seen.has(k)) return false; seen.add(k); return true; });
}

function classifyResource(r) {
  const up = r.observed === "loaded" || r.observed === "online" || r.observed === "exists";
  const down = r.observed === "not-loaded" || r.observed === "stopped" || r.observed === "absent";
  if (!r.dir) return "ambiguous";
  if (r.dir === "up") return up ? "match" : "contradict";
  if (r.dir === "down") return down ? "match" : "contradict";
  return "ambiguous";
}

function verifyMemo(file) {
  const text = readFileSync(file, "utf8");
  const explicit = extractExplicit(text);
  const evidence = [];

  if (explicit.length) {
    let pass = 0;
    for (const cmd of explicit) {
      const r = sh(cmd);
      evidence.push(`${r.ok ? "✓" : "✗"} ${cmd}`);
      if (r.ok) pass++;
    }
    return { status: pass === explicit.length ? "LANDED" : (pass === 0 ? "NO-OP" : "FAILED"),
             conf: "high", evidence };
  }

  const res = extractResources(text);
  if (!res.length) return { status: "UNVERIFIABLE", conf: "none", evidence: ["no VERIFY block, no checkable resource"] };

  const verdicts = res.map(r => { const c = classifyResource(r); evidence.push(`${r.kind}:${r.id} → ${r.observed}${r.dir ? ` (expected ${r.dir})` : ""} [${c}]`); return c; });
  if (verdicts.every(v => v === "match")) return { status: "LIKELY-LANDED", conf: "med", evidence };
  if (verdicts.some(v => v === "contradict")) return { status: "LIKELY-NOOP", conf: "med", evidence };
  return { status: "OBSERVED", conf: "low", evidence };
}

// --- run ---
let dir = opt("--dir", null);
if (!dir) dir = flag("--pending") ? join(QUEUE, "pending-approval")
        : flag("--done") ? join(QUEUE, "done")
        : join(QUEUE, "pending-approval/done");
const single = opt("--memo", null);
const wantJson = flag("--json");
const statusFilter = (opt("--status", "") || "").split(",").map(s => s.trim()).filter(Boolean);

let files = [];
if (single) files = [single];
else if (existsSync(dir)) files = readdirSync(dir).filter(f => f.endsWith(".md")).map(f => join(dir, f)).sort();
else { console.error(`dir not found: ${dir}`); process.exit(1); }

const COLOR = { LANDED: "\x1b[32m", "LIKELY-LANDED": "\x1b[92m", OBSERVED: "\x1b[36m",
  UNVERIFIABLE: "\x1b[90m", "LIKELY-NOOP": "\x1b[33m", "NO-OP": "\x1b[33m", FAILED: "\x1b[31m" };
const NC = "\x1b[0m";
const tally = {};
const rows = [];

for (const f of files) {
  const r = verifyMemo(f);
  if (statusFilter.length && !statusFilter.includes(r.status)) continue;
  tally[r.status] = (tally[r.status] || 0) + 1;
  const name = basename(f).replace(/\.md$/, "");
  rows.push({ name, ...r, file: f });
}

if (wantJson) {
  console.log(JSON.stringify({ dir, scanned: files.length, tally, rows }, null, 2));
} else {
  console.log(`\napproval-verifier — ${rows.length} item(s) in ${dir.replace(HOME, "~")}\n`);
  for (const r of rows) {
    console.log(`${COLOR[r.status] || ""}${r.status.padEnd(13)}${NC} ${r.conf.padEnd(4)} ${r.name.slice(0, 60)}`);
    for (const e of r.evidence.slice(0, 4)) console.log(`               ${e}`);
  }
  console.log("\n── summary ──");
  for (const [k, v] of Object.entries(tally).sort()) console.log(`  ${COLOR[k] || ""}${k.padEnd(13)}${NC} ${v}`);
  console.log(`\nLANDED/LIKELY-LANDED = effect confirmed.  NO-OP/LIKELY-NOOP/FAILED = did NOT land.`);
  console.log(`UNVERIFIABLE/OBSERVED = needs a VERIFY block or manual check (add  <!-- VERIFY: <shell test> -->  to the memo).`);
}

// append run ledger (skip in single/json ad-hoc mode)
if (!single && !wantJson) {
  const ts = new Date().toISOString();
  try { appendFileSync(LEDGER, rows.map(r => JSON.stringify({ ts, name: r.name, status: r.status, conf: r.conf })).join("\n") + "\n"); } catch {}
}

// --write-ledger: merge verify results back into the canonical approvals.json (closes the loop).
// Only flips POST-approval entries (executed/approved) — never a 'drafted' item.
if (flag("--write-ledger")) {
  const APPROVALS = join(QUEUE, "approvals.json");
  const MAP = { LANDED: "landed", "LIKELY-LANDED": "landed", FAILED: "failed", "NO-OP": "no-op", "LIKELY-NOOP": "no-op" };
  if (existsSync(APPROVALS)) {
    try {
      const L = JSON.parse(readFileSync(APPROVALS, "utf8"));
      const byId = new Map(rows.map(r => [r.name, r]));
      let n = 0;
      for (const e of L.entries) {
        const r = byId.get(e.id); if (!r) continue;
        e.last_checked = new Date().toISOString(); e.evidence = r.evidence;
        if (MAP[r.status] && ["executed", "approved"].includes(e.status)) { e.status = MAP[r.status]; n++; }
      }
      L.counts = L.entries.reduce((a, e) => (a[e.status] = (a[e.status] || 0) + 1, a), {});
      writeFileSync(APPROVALS, JSON.stringify(L, null, 2));
      console.log(`\n↪ wrote verify results into approvals.json (${n} status transitions)`);
    } catch (e) { console.error("ledger write failed:", e.message); }
  } else console.error("approvals.json not found — run `node ledger.mjs --build` first");
}