`scripts/doctor-progress-tick.sh`

152 lines
#!/bin/bash
# Doctor day-time progress tick.
# Snapshot the doctor project, log a JSON line, email Steve via George iff blocking.
# Wired to com.steve.doctor-progress-day (every 2h, 11/13/15/17/19/21 PT).
set -uo pipefail

PD="${PD:-/Users/macstudio3/Projects/professional-directory}"
cd "$PD" || exit 1

LOG_JSONL="$PD/data/_doctor_progress_log.jsonl"
LOG_TXT="$PD/logs/doctor-progress-tick.log"
mkdir -p "$PD/data" "$PD/logs"

ts_iso() { date -u +%Y-%m-%dT%H:%M:%SZ; }
ts_local() { date +"%H:%M:%S"; }
log() { echo "[doctor-tick $(ts_local)] $*" | tee -a "$LOG_TXT"; }

DB="doctor_professional_directory"
psqlq() { psql -d "$DB" -tA -F'|' -c "$1" 2>/dev/null; }

# ----- pm2 snapshot for the 7 pd-* agents -----
PM2_LINE=$(pm2 jlist 2>/dev/null | python3 -c '
import json, sys
try:
    d = json.load(sys.stdin)
except Exception:
    print("err|0|0|0"); sys.exit(0)
pd = [p for p in d if p.get("name","").startswith("pd-")]
online = [p for p in pd if p.get("pm2_env",{}).get("status")=="online"]
restart_total = sum(int(p.get("pm2_env",{}).get("restart_time",0)) for p in pd)
mem_mb = sum(int(p.get("monit",{}).get("memory",0)) for p in pd) // (1024*1024)
print(f"ok|{len(pd)}|{len(online)}|{restart_total}|{mem_mb}")
')
PD_TOTAL=$(echo "$PM2_LINE" | cut -d'|' -f2)
PD_ONLINE=$(echo "$PM2_LINE" | cut -d'|' -f3)
PD_RESTARTS=$(echo "$PM2_LINE" | cut -d'|' -f4)
PD_MEM=$(echo "$PM2_LINE" | cut -d'|' -f5)

# ----- pd-hawk last heartbeat (read its last log mtime) -----
HAWK_LOG="$PD/logs/pd-hawk.log"
if [ -f "$HAWK_LOG" ]; then
  HAWK_AGE_MIN=$(( ( $(date +%s) - $(stat -f %m "$HAWK_LOG") ) / 60 ))
else
  HAWK_AGE_MIN="-1"
fi

# ----- PG row counts (single round trip) -----
ROWS=$(psqlq "
SELECT
  (SELECT COUNT(*) FROM professionals),
  (SELECT COUNT(*) FROM organizations),
  (SELECT COUNT(*) FROM professional_locations),
  (SELECT COUNT(*) FROM doctor_claims WHERE status IN ('pending','submitted')),
  (SELECT COUNT(*) FROM reviews),
  (SELECT COUNT(*) FROM scrape_jobs WHERE status IN ('queued','running')),
  (SELECT COUNT(*) FROM scrape_jobs WHERE status = 'failed' AND started_at > NOW() - INTERVAL '24 hours');
")
PROS=$(echo "$ROWS" | cut -d'|' -f1); PROS=${PROS:-0}
ORGS=$(echo "$ROWS" | cut -d'|' -f2); ORGS=${ORGS:-0}
LOCS=$(echo "$ROWS" | cut -d'|' -f3); LOCS=${LOCS:-0}
CLAIMS_PENDING=$(echo "$ROWS" | cut -d'|' -f4); CLAIMS_PENDING=${CLAIMS_PENDING:-0}
REVIEWS=$(echo "$ROWS" | cut -d'|' -f5); REVIEWS=${REVIEWS:-0}
JOBS_LIVE=$(echo "$ROWS" | cut -d'|' -f6); JOBS_LIVE=${JOBS_LIVE:-0}
JOBS_FAILED_24H=$(echo "$ROWS" | cut -d'|' -f7); JOBS_FAILED_24H=${JOBS_FAILED_24H:-0}

# ----- Mockup file counts -----
MOCK_DIR="$PD/data/mockups"
MOCK_CLEAN=$(find "$MOCK_DIR" -maxdepth 1 -name '*.html' ! -name '*tainted*' 2>/dev/null | wc -l | tr -d ' ')
MOCK_TAINTED_LIVE=$(find "$MOCK_DIR" -maxdepth 1 -name '*tainted*' 2>/dev/null | wc -l | tr -d ' ')
MOCK_QUARANTINE=$(find "$MOCK_DIR/_quarantine" -name '*.html' 2>/dev/null | wc -l | tr -d ' ')

# ----- /account/claim still serving 200? -----
CLAIM_STATUS=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://localhost:9876/account/claim || echo "000")

# ----- MS1 health -----
MS1_STATUS=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://192.168.1.133:11434/api/tags || echo "000")

# ----- Decide if anything is BLOCKING (warrants email) -----
ALERTS=()
[ "$PD_ONLINE" = "0" ] || [ "$PD_TOTAL" = "0" ] && ALERTS+=("pd-* agents not visible (total=$PD_TOTAL online=$PD_ONLINE)")
[ -n "$PD_TOTAL" ] && [ "$PD_TOTAL" != "0" ] && [ "$PD_ONLINE" != "$PD_TOTAL" ] && ALERTS+=("pd-* not all online: $PD_ONLINE/$PD_TOTAL")
[ "$CLAIM_STATUS" != "200" ] && ALERTS+=("/account/claim returned $CLAIM_STATUS (expected 200)")
[ "$MS1_STATUS" != "200" ] && ALERTS+=("MS1 ollama returned $MS1_STATUS (expected 200)")
[ "$MOCK_TAINTED_LIVE" != "0" ] && ALERTS+=("$MOCK_TAINTED_LIVE tainted mockups in main dir (expected 0; should be quarantined)")
[ "$JOBS_FAILED_24H" -gt 50 ] 2>/dev/null && ALERTS+=("scrape_jobs failed in last 24h: $JOBS_FAILED_24H")
[ "$HAWK_AGE_MIN" != "-1" ] && [ "$HAWK_AGE_MIN" -gt 90 ] 2>/dev/null && ALERTS+=("pd-hawk log stale ($HAWK_AGE_MIN min)")

# ----- Append JSONL log line -----
ALERT_JSON="[]"
if [ ${#ALERTS[@]} -gt 0 ]; then
  ALERT_JSON=$(printf '%s\n' "${ALERTS[@]}" | python3 -c 'import json,sys; print(json.dumps([l.rstrip() for l in sys.stdin if l.strip()]))')
fi

cat >> "$LOG_JSONL" <<JSON
{"ts":"$(ts_iso)","pd":{"total":$PD_TOTAL,"online":$PD_ONLINE,"restarts":$PD_RESTARTS,"mem_mb":$PD_MEM},"hawk_age_min":$HAWK_AGE_MIN,"pg":{"pros":$PROS,"orgs":$ORGS,"locs":$LOCS,"claims_pending":$CLAIMS_PENDING,"reviews":$REVIEWS,"jobs_live":$JOBS_LIVE,"jobs_failed_24h":$JOBS_FAILED_24H},"mockups":{"clean":$MOCK_CLEAN,"tainted_live":$MOCK_TAINTED_LIVE,"quarantine":$MOCK_QUARANTINE},"http":{"claim_page":"$CLAIM_STATUS","ms1_ollama":"$MS1_STATUS"},"alerts":$ALERT_JSON}
JSON

log "snap pd=$PD_ONLINE/$PD_TOTAL pg=$PROS/$ORGS/$LOCS claims=$CLAIMS_PENDING mockups=$MOCK_CLEAN/$MOCK_TAINTED_LIVE/Q$MOCK_QUARANTINE claim=$CLAIM_STATUS MS1=$MS1_STATUS alerts=${#ALERTS[@]}"

# ----- Email George iff alerts (rate-limited: only if previous tick had no alerts OR it's been >4h since last email) -----
LAST_EMAIL_FILE="$PD/logs/.doctor-progress-last-email"
SHOULD_EMAIL=0
if [ ${#ALERTS[@]} -gt 0 ]; then
  if [ -f "$LAST_EMAIL_FILE" ]; then
    LAST_AGE=$(( $(date +%s) - $(stat -f %m "$LAST_EMAIL_FILE") ))
    [ "$LAST_AGE" -gt 14400 ] && SHOULD_EMAIL=1
  else
    SHOULD_EMAIL=1
  fi
fi

if [ "$SHOULD_EMAIL" = "1" ]; then
  ALERT_BODY=$(printf '%s\n' "${ALERTS[@]}")
  BODY="Doctor project tick @ $(date +"%Y-%m-%d %H:%M %Z")

ALERTS:
$ALERT_BODY

State:
- pd-* agents: $PD_ONLINE/$PD_TOTAL online ($PD_RESTARTS total restarts, ${PD_MEM}MB)
- pd-hawk last heartbeat: $HAWK_AGE_MIN min ago
- PG: $PROS pros / $ORGS orgs / $LOCS locs
- Claims pending: $CLAIMS_PENDING
- Mockups: clean=$MOCK_CLEAN tainted_live=$MOCK_TAINTED_LIVE quarantined=$MOCK_QUARANTINE
- /account/claim: $CLAIM_STATUS  |  MS1 ollama: $MS1_STATUS
- scrape_jobs: live=$JOBS_LIVE failed_24h=$JOBS_FAILED_24H

Auto-generated by com.steve.doctor-progress-day"

  python3 -c '
import json, urllib.request, sys, os
body = sys.stdin.read()
payload = json.dumps({
  "to": "steveabramsdesigns@gmail.com",
  "subject": "Doctor tick alerts — " + os.environ.get("WHEN", ""),
  "body": body
}).encode()
req = urllib.request.Request(
  "http://localhost:9850/api/send?account=info",
  data=payload,
  headers={"Content-Type": "application/json"}
)
try:
  r = urllib.request.urlopen(req, timeout=10)
  print("george:", r.status)
except Exception as e:
  print("george-err:", e, file=sys.stderr)
' WHEN="$(date +"%Y-%m-%d %H:%M")" <<<"$BODY" >> "$LOG_TXT" 2>&1 && touch "$LAST_EMAIL_FILE"
fi

exit 0