Add transition-only canary alert wrapper (CNCP card + George email on NEW problem, silent otherwise; --selftest+DRY_RUN verified); point canary plist at it — enable is gated

b8b0d82039a2c3b1540eabbf2d98486b9def58bc · 2026-08-12 09:26:44 -0700 · Steve Abrams
Files touched

A scripts/canary-alert.sh
M scripts/launchd/com.steve.la-data-canary.plist
Diff

commit b8b0d82039a2c3b1540eabbf2d98486b9def58bc
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Aug 12 09:26:44 2026 -0700

    Add transition-only canary alert wrapper (CNCP card + George email on NEW problem, silent otherwise; --selftest+DRY_RUN verified); point canary plist at it — enable is gated
---
 scripts/canary-alert.sh                        | 82 ++++++++++++++++++++++++++
 scripts/launchd/com.steve.la-data-canary.plist |  4 +-
 2 files changed, 84 insertions(+), 2 deletions(-)

diff --git a/scripts/canary-alert.sh b/scripts/canary-alert.sh
new file mode 100755
index 0000000..46ea256
--- /dev/null
+++ b/scripts/canary-alert.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+# canary-alert.sh — run the LA data freshness canary, then alert ONLY on a WORSENING
+# transition (a NEW actionable problem appears), matching the DW canary doctrine:
+#   • CNCP parking-lot card + George email on worsening; silent otherwise.
+#   • Steady-state, recovery (a problem clears), and first-run are all SILENT.
+#   • Known-broken sources (gis_zoning) are excluded so they can't pin a perpetual alert.
+#
+# Design note: the canary's steady verdict is WARN (gis_zoning is known-broken), so we do
+# NOT alert on the coarse verdict — we diff the SET of actionable problems
+# {name:status} across runs and alert when a NEW one shows up. That catches a feed going
+# STALE (the refresh job died) even while gis_zoning sits at BROKEN.
+#
+# Modes:
+#   (default)         run canary, evaluate transition, alert if worsening, persist state.
+#   DRY_RUN=1         compute + PRINT what it would alert; NO CNCP post, NO email, NO state write.
+#   --selftest        synthetic prev={}, current={la_building_permits_raw:STALE}; implies DRY.
+# $0 local (the canary reads Postgres; alerts hit localhost CNCP + George).
+set -uo pipefail
+cd "$(dirname "$0")/.."                       # project root
+DATA="tmp/canary"; STATE="$DATA/alert-state.json"; LATEST="$DATA/latest.json"; LOG="$DATA/alert.log"
+mkdir -p "$DATA"
+log(){ echo "[$(date -Iseconds)] $1" | tee -a "$LOG"; }
+
+SELFTEST=0; [ "${1:-}" = "--selftest" ] && { SELFTEST=1; DRY_RUN=1; }
+DRY_RUN="${DRY_RUN:-0}"
+
+# ── obtain current problem set ──
+if [ "$SELFTEST" = "1" ]; then
+  CUR='["la_building_permits_raw:STALE"]'
+  VERDICT="WARN"
+  log "selftest: synthetic current problem set = $CUR"
+else
+  node scripts/canary-freshness.js > "$DATA/report.txt" 2>>"$LOG" || { log "canary run failed"; exit 1; }
+  [ -f "$LATEST" ] || { log "no canary latest.json"; exit 1; }
+  VERDICT=$(jq -r '.verdict' "$LATEST")
+  # actionable problems = non-OK result tables + non-OK/non-BROKEN sources (drop known-broken)
+  CUR=$(jq -c '[ (.results[]  | select(.status!="OK")                    | "\(.table):\(.status)"),
+                 (.sources[]  | select(.status!="OK" and .status!="BROKEN") | "\(.source):\(.status)") ]' "$LATEST")
+  log "verdict=$VERDICT  problems=$CUR"
+fi
+
+# ── load prior problem set; first run establishes baseline silently ──
+FIRST=0; [ -f "$STATE" ] || FIRST=1
+PREV=$(jq -c '.problems // []' "$STATE" 2>/dev/null || echo '[]')
+[ "$SELFTEST" = "1" ] && { PREV='[]'; FIRST=0; }
+
+# new problems = in CUR but not in PREV
+NEW=$(jq -cn --argjson cur "$CUR" --argjson prev "$PREV" '$cur - $prev')
+NEWCOUNT=$(echo "$NEW" | jq 'length')
+
+persist(){ [ "$DRY_RUN" = "1" ] && return 0; jq -n --argjson p "$CUR" --arg ts "$(date -Iseconds)" '{problems:$p, at:$ts}' > "$STATE"; }
+
+if [ "$FIRST" = "1" ]; then log "first run — baseline established, silent"; persist; exit 0; fi
+if [ "$NEWCOUNT" -eq 0 ]; then log "no NEW problems ($PREV -> $CUR) — silent"; persist; exit 0; fi
+
+# ── worsening: a new actionable problem appeared. Alert. ──
+SUMMARY=$(echo "$NEW" | jq -r 'join("; ")')
+log "WORSENING — new problems: $SUMMARY"
+NOTE="[LA DATA CANARY $(date +%Y-%m-%d) $VERDICT] New freshness/rowcount regression on the LA public-data platform: $SUMMARY. Feeds buildingpermits.agentabrams.com. Check the 04:30 refresh job (com.steve.la-data-refresh) + tmp/ingest.log."
+
+if [ "$DRY_RUN" = "1" ]; then
+  echo "── DRY_RUN: would POST CNCP + email George ──"
+  echo "CNCP note: $NOTE"
+  echo "Email subject: ⚠ LA data canary — $VERDICT — new: $SUMMARY ($(date +%Y-%m-%d))"
+  exit 0
+fi
+
+CNCP="${CNCP_URL:-http://localhost:3333}"
+curl -sS --max-time 10 "$CNCP/api/parking-lot" -H 'Content-Type: application/json' \
+  -d "$(jq -n --arg u "https://buildingpermits.agentabrams.com" --arg note "$NOTE" '{url:$u, note:$note}')" >/dev/null 2>&1 \
+  && log "CNCP posted" || log "CNCP post failed"
+
+if [ -f "$HOME/.claude/skills/_shared/george-send.sh" ]; then
+  . "$HOME/.claude/skills/_shared/george-send.sh"
+  TO="${LA_CANARY_TO:-steve@designerwallcoverings.com}"
+  BODY="<div style=\"font-family:-apple-system,Helvetica,sans-serif;color:#222\"><h3 style=\"color:#b23b3b\">⚠ LA data canary — $VERDICT</h3><p style=\"color:#888;font-size:13px\">$(date -Iseconds) · buildingpermits.agentabrams.com</p><div style=\"margin:8px 0;padding:8px 12px;border:1px solid #eee;border-radius:6px\">New regression: $SUMMARY</div><p style=\"font-size:13px\">A feed stopped refreshing or a row count dropped. Check the 04:30 refresh job (<code>com.steve.la-data-refresh</code>) and <code>tmp/ingest.log</code>.</p></div>"
+  RESP="$(george_send steve-office "$TO" "⚠ LA data canary — $VERDICT — new: $SUMMARY ($(date +%Y-%m-%d))" "$BODY")"
+  echo "$RESP" > "$DATA/email-last.json"
+  echo "$RESP" | grep -q '"success":true' && log "alert emailed to $TO" || log "email failed — see email-last.json"
+fi
+persist
+log "surfaced $VERDICT"
diff --git a/scripts/launchd/com.steve.la-data-canary.plist b/scripts/launchd/com.steve.la-data-canary.plist
index cad0bf3..6655b47 100644
--- a/scripts/launchd/com.steve.la-data-canary.plist
+++ b/scripts/launchd/com.steve.la-data-canary.plist
@@ -6,8 +6,8 @@
     <string>com.steve.la-data-canary</string>
     <key>ProgramArguments</key>
     <array>
-        <string>/opt/homebrew/bin/node</string>
-        <string>/Users/macstudio3/Projects/la-socrata-ingester/scripts/canary-freshness.js</string>
+        <string>/bin/bash</string>
+        <string>/Users/macstudio3/Projects/la-socrata-ingester/scripts/canary-alert.sh</string>
     </array>
     <key>WorkingDirectory</key>
     <string>/Users/macstudio3/Projects/la-socrata-ingester</string>
← cc6b19f Archive la-permits-live-refresh plist for one-copy rollback · back to La Socrata Ingester · waf: period 10 (plan-allowed) — 20 req/10s = 120/min ceiling 15c6ec1 →