`scripts/canary-alert.sh`

92 lines
#!/bin/bash
# canary-alert.sh — run the LA data freshness canary, then alert ONLY on a WORSENING
# transition (a NEW actionable problem appears), matching the DW canary doctrine:
#   • CNCP parking-lot card + George email on worsening; silent otherwise.
#   • Steady-state, recovery (a problem clears), and first-run are all SILENT.
#   • Known-broken sources (gis_zoning) are excluded so they can't pin a perpetual alert.
#
# Design note: the canary's steady verdict is WARN (gis_zoning is known-broken), so we do
# NOT alert on the coarse verdict — we diff the SET of actionable problems
# {name:status} across runs and alert when a NEW one shows up. That catches a feed going
# STALE (the refresh job died) even while gis_zoning sits at BROKEN.
#
# Modes:
#   (default)         run canary, evaluate transition, alert if worsening, persist state.
#   DRY_RUN=1         compute + PRINT what it would alert; NO CNCP post, NO email, NO state write.
#   --selftest        synthetic prev={}, current={la_building_permits_raw:STALE}; implies DRY.
# $0 local (the canary reads Postgres; alerts hit localhost CNCP + George).
set -uo pipefail
cd "$(dirname "$0")/.."                       # project root
DATA="tmp/canary"; STATE="$DATA/alert-state.json"; LATEST="$DATA/latest.json"; LOG="$DATA/alert.log"
mkdir -p "$DATA"
log(){ echo "[$(date -Iseconds)] $1" | tee -a "$LOG"; }

SELFTEST=0; TESTFIRE=0
case "${1:-}" in
  --selftest) SELFTEST=1; DRY_RUN=1 ;;
  --testfire) TESTFIRE=1 ;;   # fire ONE real alert (CNCP+George), synthetic, does NOT touch armed state
esac
DRY_RUN="${DRY_RUN:-0}"

# ── obtain current problem set ──
if [ "$TESTFIRE" = "1" ]; then
  CUR='["TESTFIRE:delivery-check"]'; VERDICT="WARN"
  log "testfire: firing ONE real alert (synthetic) — armed state untouched"
elif [ "$SELFTEST" = "1" ]; then
  CUR='["la_building_permits_raw:STALE"]'
  VERDICT="WARN"
  log "selftest: synthetic current problem set = $CUR"
else
  node scripts/canary-freshness.js > "$DATA/report.txt" 2>>"$LOG" || { log "canary run failed"; exit 1; }
  [ -f "$LATEST" ] || { log "no canary latest.json"; exit 1; }
  VERDICT=$(jq -r '.verdict' "$LATEST")
  # actionable problems = non-OK result tables + non-OK/non-BROKEN sources (drop known-broken)
  CUR=$(jq -c '[ (.results[]  | select(.status!="OK")                    | "\(.table):\(.status)"),
                 (.sources[]  | select(.status!="OK" and .status!="BROKEN") | "\(.source):\(.status)") ]' "$LATEST")
  log "verdict=$VERDICT  problems=$CUR"
fi

# ── load prior problem set; first run establishes baseline silently ──
FIRST=0; [ -f "$STATE" ] || FIRST=1
PREV=$(jq -c '.problems // []' "$STATE" 2>/dev/null || echo '[]')
[ "$SELFTEST" = "1" ] && { PREV='[]'; FIRST=0; }
[ "$TESTFIRE" = "1" ] && { PREV='[]'; FIRST=0; }

# new problems = in CUR but not in PREV
NEW=$(jq -cn --argjson cur "$CUR" --argjson prev "$PREV" '$cur - $prev')
NEWCOUNT=$(echo "$NEW" | jq 'length')

persist(){ { [ "$DRY_RUN" = "1" ] || [ "$TESTFIRE" = "1" ]; } && return 0; jq -n --argjson p "$CUR" --arg ts "$(date -Iseconds)" '{problems:$p, at:$ts}' > "$STATE"; }

if [ "$FIRST" = "1" ]; then log "first run — baseline established, silent"; persist; exit 0; fi
if [ "$NEWCOUNT" -eq 0 ]; then log "no NEW problems ($PREV -> $CUR) — silent"; persist; exit 0; fi

# ── worsening: a new actionable problem appeared. Alert. ──
SUMMARY=$(echo "$NEW" | jq -r 'join("; ")')
log "WORSENING — new problems: $SUMMARY"
PFX=""; [ "$TESTFIRE" = "1" ] && PFX="[TEST — canary delivery check, ignore] "
NOTE="${PFX}[LA DATA CANARY $(date +%Y-%m-%d) $VERDICT] New freshness/rowcount regression on the LA public-data platform: $SUMMARY. Feeds buildingpermits.agentabrams.com. Check the 04:30 refresh job (com.steve.la-data-refresh) + tmp/ingest.log."

if [ "$DRY_RUN" = "1" ]; then
  echo "── DRY_RUN: would POST CNCP + email George ──"
  echo "CNCP note: $NOTE"
  echo "Email subject: ⚠ LA data canary — $VERDICT — new: $SUMMARY ($(date +%Y-%m-%d))"
  exit 0
fi

CNCP="${CNCP_URL:-http://localhost:3333}"
curl -sS --max-time 10 "$CNCP/api/parking-lot" -H 'Content-Type: application/json' \
  -d "$(jq -n --arg u "https://buildingpermits.agentabrams.com" --arg note "$NOTE" '{url:$u, note:$note}')" >/dev/null 2>&1 \
  && log "CNCP posted" || log "CNCP post failed"

if [ -f "$HOME/.claude/skills/_shared/george-send.sh" ]; then
  . "$HOME/.claude/skills/_shared/george-send.sh"
  TO="${LA_CANARY_TO:-steve@designerwallcoverings.com}"
  BODY="<div style=\"font-family:-apple-system,Helvetica,sans-serif;color:#222\"><h3 style=\"color:#b23b3b\">⚠ LA data canary — $VERDICT</h3><p style=\"color:#888;font-size:13px\">$(date -Iseconds) · buildingpermits.agentabrams.com</p><div style=\"margin:8px 0;padding:8px 12px;border:1px solid #eee;border-radius:6px\">New regression: $SUMMARY</div><p style=\"font-size:13px\">A feed stopped refreshing or a row count dropped. Check the 04:30 refresh job (<code>com.steve.la-data-refresh</code>) and <code>tmp/ingest.log</code>.</p></div>"
  RESP="$(george_send steve-office "$TO" "${PFX}⚠ LA data canary — $VERDICT — new: $SUMMARY ($(date +%Y-%m-%d))" "$BODY")"
  echo "$RESP" > "$DATA/email-last.json"
  echo "$RESP" | grep -q '"success":true' && log "alert emailed to $TO" || log "email failed — see email-last.json"
fi
persist
log "surfaced $VERDICT"